handle errors on messages so it doesnt crash, add new embedding

Author: lalalune

packages/client-discord/src/actions/download_media.ts

@@ -133,10 +133,6 @@ export default {
                     console.error(
                         "Max retries reached. Failed to send message with attachment."
                     );
-                    await callback({
-                        ...response,
-                        text: "Sorry, I encountered an error while trying to send the video attachment. Please try again later.",
-                    });
                     break;
                 }
 

packages/client-discord/src/messages.ts

@@ -595,9 +595,7 @@ export class MessageManager {
                 await this.voiceManager.playAudioStream(userId, audioStream);
             } else {
                 // For text channels, send the error message
-                await message.reply(
-                    "Sorry, I encountered an error while processing your request."
-                );
+                console.error("Error sending message:", error);
             }
         }
     }

packages/client-telegram/src/messageManager.ts

@@ -474,9 +474,7 @@ export class MessageManager {
             );
         } catch (error) {
             console.error("❌ Error handling message:", error);
-            await ctx.reply(
-                "Sorry, I encountered an error while processing your request."
-            );
+            console.error("Error sending message:", error);
         }
     }
 }

packages/core/package.json

@@ -15,7 +15,6 @@
         "watch": "tsc --watch",
         "dev": "tsup --format esm --dts --watch",
         "build:docs": "cd docs && pnpm run build",
-        "postinstall": "npx playwright install-deps && npx playwright install",
         "test": "jest --runInBand",
         "test:watch": "jest --runInBand --watch"
     },
@@ -43,9 +42,9 @@
         "eslint-plugin-prettier": "5.2.1",
         "jest": "29.7.0",
         "lint-staged": "15.2.10",
-        "prettier": "3.3.3",
         "nodemon": "3.1.7",
         "pm2": "5.4.2",
+        "prettier": "3.3.3",
         "rimraf": "6.0.1",
         "rollup": "2.79.2",
         "ts-jest": "29.2.5",
@@ -64,6 +63,7 @@
         "@types/uuid": "^10.0.0",
         "ai": "^3.4.23",
         "anthropic-vertex-ai": "^1.0.0",
+        "fastembed": "^1.14.1",
         "gaxios": "6.7.1",
         "glob": "11.0.0",
         "js-sha1": "0.7.0",

packages/core/src/embedding.ts

@@ -1,10 +1,13 @@
+import path from "path";
 import models from "./models.ts";
 import {
     IAgentRuntime,
     ITextGenerationService,
     ModelProviderName,
     ServiceType,
 } from "./types.ts";
+import fs from "fs";
+import { EmbeddingModel, FlagEmbedding } from "fastembed";
 
 /**
  * Send a message to the OpenAI API for embedding.
@@ -20,21 +23,35 @@ export async function embed(runtime: IAgentRuntime, input: string) {
         runtime.character.modelProvider !== ModelProviderName.OPENAI &&
         runtime.character.modelProvider !== ModelProviderName.OLLAMA
     ) {
-        const service = runtime.getService<ITextGenerationService>(
-            ServiceType.TEXT_GENERATION
-        );
+
+        // make sure to trim tokens to 8192
+
+        const embeddingModel = await FlagEmbedding.init({
+            model: EmbeddingModel.BGEBaseEN
+        });
 
-        const instance = service?.getInstance();
+        const embedding: number[] = await embeddingModel.queryEmbed(input);
+        console.log("Embedding dimensions: ", embedding.length);
+        return embedding;
 
-        if (instance) {
-            return await instance.getEmbeddingResponse(input);
-        }
+        // commented out the text generation service that uses llama
+        // const service = runtime.getService<ITextGenerationService>(
+        //     ServiceType.TEXT_GENERATION
+        // );
+        
+        // const instance = service?.getInstance();
+
+        // if (instance) {
+        //     return await instance.getEmbeddingResponse(input);
+        // }
     }
+
+    // TODO: Fix retrieveCachedEmbedding
     // Check if we already have the embedding in the lore
-    // const cachedEmbedding = await retrieveCachedEmbedding(runtime, input);
-    // if (cachedEmbedding) {
-    //     return cachedEmbedding;
-    // }
+    const cachedEmbedding = await retrieveCachedEmbedding(runtime, input);
+    if (cachedEmbedding) {
+        return cachedEmbedding;
+    }
 
     const requestOptions = {
         method: "POST",
@@ -48,7 +65,8 @@ export async function embed(runtime: IAgentRuntime, input: string) {
         body: JSON.stringify({
             input,
             model: embeddingModel,
-            length: 1536,
+            length: 768, // we are squashing dimensions to 768 for openai, even thought the model supports 1536
+            // -- this is ok for matryoshka embeddings but longterm, we might want to support 1536
         }),
     };
     try {

packages/plugin-node/package.json

@@ -63,7 +63,8 @@
     },
     "scripts": {
         "build": "tsup --format esm --dts",
-        "dev": "tsup --watch"
+        "dev": "tsup --watch",
+        "postinstall": "npx playwright install-deps && npx playwright install"
     },
     "peerDependencies": {
         "whatwg-url": "7.1.0",