|
| 1 | +import bodyParser from "body-parser"; |
| 2 | +import express from "express"; |
| 3 | +import { composeContext } from "../../core/context.ts"; |
| 4 | +import { AgentRuntime } from "../../core/runtime.ts"; |
| 5 | +import { Content, Memory, ModelClass, State } from "../../core/types.ts"; |
| 6 | +import { stringToUuid } from "../../core/uuid.ts"; |
| 7 | +import cors from "cors"; |
| 8 | +import { messageCompletionFooter } from "../../core/parsing.ts"; |
| 9 | +import multer, { File } from "multer"; |
| 10 | +import { Request as ExpressRequest } from "express"; |
| 11 | +import { generateMessageResponse } from "../../core/generation.ts"; |
| 12 | +import { |
| 13 | + generateCaption, |
| 14 | + generateImage, |
| 15 | +} from "../../actions/imageGenerationUtils.ts"; |
| 16 | + |
| 17 | +const upload = multer({ storage: multer.memoryStorage() }); |
| 18 | + |
| 19 | +export const messageHandlerTemplate = |
| 20 | + // {{goals}} |
| 21 | + `# Action Examples |
| 22 | +{{actionExamples}} |
| 23 | +(Action examples are for reference only. Do not use the information from them in your response.) |
| 24 | +
|
| 25 | +# Task: Generate dialog and actions for the character {{agentName}}. |
| 26 | +About {{agentName}}: |
| 27 | +{{bio}} |
| 28 | +{{lore}} |
| 29 | +
|
| 30 | +{{providers}} |
| 31 | +
|
| 32 | +{{attachments}} |
| 33 | +
|
| 34 | +# Capabilities |
| 35 | +Note that {{agentName}} is capable of reading/seeing/hearing various forms of media, including images, videos, audio, plaintext and PDFs. Recent attachments have been included above under the "Attachments" section. |
| 36 | +
|
| 37 | +{{messageDirections}} |
| 38 | +
|
| 39 | +{{recentMessages}} |
| 40 | +
|
| 41 | +{{actions}} |
| 42 | +
|
| 43 | +# Instructions: Write the next message for {{agentName}}. Ignore "action". |
| 44 | +` + messageCompletionFooter; |
| 45 | + |
| 46 | +export interface SimliClientConfig { |
| 47 | + apiKey: string; |
| 48 | + faceID: string; |
| 49 | + handleSilence: boolean; |
| 50 | + videoRef: any; |
| 51 | + audioRef: any; |
| 52 | +} |
| 53 | +class DirectClient { |
| 54 | + private app: express.Application; |
| 55 | + private agents: Map<string, AgentRuntime>; |
| 56 | + |
| 57 | + constructor() { |
| 58 | + this.app = express(); |
| 59 | + this.app.use(cors()); |
| 60 | + this.agents = new Map(); |
| 61 | + |
| 62 | + this.app.use(bodyParser.json()); |
| 63 | + this.app.use(bodyParser.urlencoded({ extended: true })); |
| 64 | + |
| 65 | + // Define an interface that extends the Express Request interface |
| 66 | + interface CustomRequest extends ExpressRequest { |
| 67 | + file: File; |
| 68 | + } |
| 69 | + |
| 70 | + // Update the route handler to use CustomRequest instead of express.Request |
| 71 | + this.app.post( |
| 72 | + "/:agentId/whisper", |
| 73 | + upload.single("file"), |
| 74 | + async (req: CustomRequest, res: express.Response) => { |
| 75 | + const audioFile = req.file; // Access the uploaded file using req.file |
| 76 | + const agentId = req.params.agentId; |
| 77 | + |
| 78 | + if (!audioFile) { |
| 79 | + res.status(400).send("No audio file provided"); |
| 80 | + return; |
| 81 | + } |
| 82 | + |
| 83 | + let runtime = this.agents.get(agentId); |
| 84 | + |
| 85 | + // if runtime is null, look for runtime with the same name |
| 86 | + if (!runtime) { |
| 87 | + runtime = Array.from(this.agents.values()).find( |
| 88 | + (a) => |
| 89 | + a.character.name.toLowerCase() === |
| 90 | + agentId.toLowerCase() |
| 91 | + ); |
| 92 | + } |
| 93 | + |
| 94 | + if (!runtime) { |
| 95 | + res.status(404).send("Agent not found"); |
| 96 | + return; |
| 97 | + } |
| 98 | + |
| 99 | + const formData = new FormData(); |
| 100 | + const audioBlob = new Blob([audioFile.buffer], { |
| 101 | + type: audioFile.mimetype, |
| 102 | + }); |
| 103 | + formData.append("file", audioBlob, audioFile.originalname); |
| 104 | + formData.append("model", "whisper-1"); |
| 105 | + |
| 106 | + const response = await fetch( |
| 107 | + "https://api.openai.com/v1/audio/transcriptions", |
| 108 | + { |
| 109 | + method: "POST", |
| 110 | + headers: { |
| 111 | + Authorization: `Bearer ${runtime.token}`, |
| 112 | + }, |
| 113 | + body: formData, |
| 114 | + } |
| 115 | + ); |
| 116 | + |
| 117 | + const data = await response.json(); |
| 118 | + res.json(data); |
| 119 | + } |
| 120 | + ); |
| 121 | + |
| 122 | + this.app.post( |
| 123 | + "/:agentId/message", |
| 124 | + async (req: express.Request, res: express.Response) => { |
| 125 | + const agentId = req.params.agentId; |
| 126 | + const roomId = stringToUuid( |
| 127 | + req.body.roomId ?? "default-room-" + agentId |
| 128 | + ); |
| 129 | + const userId = stringToUuid(req.body.userId ?? "user"); |
| 130 | + |
| 131 | + let runtime = this.agents.get(agentId); |
| 132 | + |
| 133 | + // if runtime is null, look for runtime with the same name |
| 134 | + if (!runtime) { |
| 135 | + runtime = Array.from(this.agents.values()).find( |
| 136 | + (a) => |
| 137 | + a.character.name.toLowerCase() === |
| 138 | + agentId.toLowerCase() |
| 139 | + ); |
| 140 | + } |
| 141 | + |
| 142 | + if (!runtime) { |
| 143 | + res.status(404).send("Agent not found"); |
| 144 | + return; |
| 145 | + } |
| 146 | + |
| 147 | + await runtime.ensureConnection( |
| 148 | + userId, |
| 149 | + roomId, |
| 150 | + req.body.userName, |
| 151 | + req.body.name, |
| 152 | + "direct" |
| 153 | + ); |
| 154 | + |
| 155 | + const text = req.body.text; |
| 156 | + const messageId = stringToUuid(Date.now().toString()); |
| 157 | + |
| 158 | + const content: Content = { |
| 159 | + text, |
| 160 | + attachments: [], |
| 161 | + source: "direct", |
| 162 | + inReplyTo: undefined, |
| 163 | + }; |
| 164 | + |
| 165 | + const userMessage = { content, userId, roomId, agentId: runtime.agentId }; |
| 166 | + |
| 167 | + const memory: Memory = { |
| 168 | + id: messageId, |
| 169 | + agentId: runtime.agentId, |
| 170 | + userId, |
| 171 | + roomId, |
| 172 | + content, |
| 173 | + createdAt: Date.now(), |
| 174 | + }; |
| 175 | + |
| 176 | + await runtime.messageManager.createMemory(memory); |
| 177 | + |
| 178 | + const state = (await runtime.composeState(userMessage, { |
| 179 | + agentName: runtime.character.name, |
| 180 | + })) as State; |
| 181 | + |
| 182 | + const context = composeContext({ |
| 183 | + state, |
| 184 | + template: messageHandlerTemplate, |
| 185 | + }); |
| 186 | + |
| 187 | + const response = await generateMessageResponse({ |
| 188 | + runtime: runtime, |
| 189 | + context, |
| 190 | + modelClass: ModelClass.SMALL, |
| 191 | + }); |
| 192 | + |
| 193 | + // save response to memory |
| 194 | + const responseMessage = { |
| 195 | + ...userMessage, |
| 196 | + userId: runtime.agentId, |
| 197 | + content: response, |
| 198 | + }; |
| 199 | + |
| 200 | + await runtime.messageManager.createMemory(responseMessage); |
| 201 | + |
| 202 | + if (!response) { |
| 203 | + res.status(500).send( |
| 204 | + "No response from generateMessageResponse" |
| 205 | + ); |
| 206 | + return; |
| 207 | + } |
| 208 | + |
| 209 | + let message = null as Content | null; |
| 210 | + |
| 211 | + const result = await runtime.processActions( |
| 212 | + memory, |
| 213 | + [responseMessage], |
| 214 | + state, |
| 215 | + async (newMessages) => { |
| 216 | + message = newMessages; |
| 217 | + return [memory]; |
| 218 | + } |
| 219 | + ) |
| 220 | + |
| 221 | + if (message) { |
| 222 | + res.json([message, response]); |
| 223 | + } else { |
| 224 | + res.json([response]); |
| 225 | + } |
| 226 | + |
| 227 | + } |
| 228 | + ); |
| 229 | + |
| 230 | + this.app.post( |
| 231 | + "/:agentId/image", |
| 232 | + async (req: express.Request, res: express.Response) => { |
| 233 | + const agentId = req.params.agentId; |
| 234 | + const agent = this.agents.get(agentId); |
| 235 | + if (!agent) { |
| 236 | + res.status(404).send("Agent not found"); |
| 237 | + return; |
| 238 | + } |
| 239 | + |
| 240 | + const images = await generateImage({ ...req.body }, agent); |
| 241 | + const imagesRes: { image: string; caption: string }[] = []; |
| 242 | + if (images.data && images.data.length > 0) { |
| 243 | + for (let i = 0; i < images.data.length; i++) { |
| 244 | + const caption = await generateCaption( |
| 245 | + { imageUrl: images.data[i] }, |
| 246 | + agent |
| 247 | + ); |
| 248 | + imagesRes.push({ |
| 249 | + image: images.data[i], |
| 250 | + caption: caption.title, |
| 251 | + }); |
| 252 | + } |
| 253 | + } |
| 254 | + res.json({ images: imagesRes }); |
| 255 | + } |
| 256 | + ); |
| 257 | + } |
| 258 | + |
| 259 | + public registerAgent(runtime: AgentRuntime) { |
| 260 | + this.agents.set(runtime.agentId, runtime); |
| 261 | + } |
| 262 | + |
| 263 | + public unregisterAgent(runtime: AgentRuntime) { |
| 264 | + this.agents.delete(runtime.agentId); |
| 265 | + } |
| 266 | + |
| 267 | + public start(port: number) { |
| 268 | + this.app.listen(port, () => { |
| 269 | + console.log(`Server running at http://localhost:${port}/`); |
| 270 | + }); |
| 271 | + } |
| 272 | +} |
| 273 | + |
| 274 | +export { DirectClient }; |
0 commit comments