removing duplicates

Author: mike dupont

packages/adapter-sqlite/src/index.ts

@@ -424,7 +424,7 @@ export class SqliteDatabaseAdapter
             ORDER BY levenshtein_score ASC
             LIMIT ?
         `;
-
+        console.log("similarity",sql)
         const rows = this.db
             .prepare(sql)
             .all(
@@ -439,7 +439,7 @@ export class SqliteDatabaseAdapter
                 opts.query_input,
                 opts.query_match_count
             ) as { embedding: Buffer; levenshtein_score: number }[];
-
+        console.log("found these",rows)
         return rows.map((row) => ({
             embedding: Array.from(new Float32Array(row.embedding as Buffer)),
             levenshtein_score: row.levenshtein_score,

packages/core/src/embedding.ts

@@ -295,6 +295,7 @@ export async function embed(runtime: IAgentRuntime, input: string) {
         }
         if (! runtime.messageManager) {
             elizaLogger.log("No message manager");
+            throw Error("No message manager");
             return null;
         }
 

scripts/jsdoc-automation/src/AIService/AIService.ts

@@ -326,6 +326,18 @@ export class AIService {
         throw error;
     }
 }
+function contentToUuid(content:string):UUID {
+    // Step 1: Create a SHA-1 hash of the content
+    const hash = crypto.createHash('sha1')
+                      .update(content)
+                      .digest('hex'); // Outputs a 40-character hex string
+
+    // Step 2: Format the hash into a UUID-like structure
+    // UUID format: 8-4-4-4-12 (e.g., 550e8400-e29b-41d4-a716-446655440000)
+    const uuid = `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20, 32)}`;
+
+    return uuid as `${string}-${string}-${string}-${string}-${string}`;
+}
 
 async function processChunk(prompt: string, manager: string, runtime: AgentRuntime): Promise<string> {
     console.log("process chunk");
@@ -336,15 +348,30 @@ async function processChunk(prompt: string, manager: string, runtime: AgentRunti
             console.log("memoryManagers",runtime.memoryManagers);
             throw new Error(`Memory manager not found {manager}`);
         }
-        const memory = await mm.addEmbeddingToMemory({
-            agentId: runtime.agentId,
-            userId: runtime.agentId, //|| runtime.userId,
-            content: { text: prompt },
-            roomId: runtime.agentId,
-            embedding: await embed(runtime, prompt),
-        });
-        await mm.createMemory(memory);
-        return "Success";
+
+        const memId = contentToUuid (prompt);
+        const existingResponse =
+                        await mm.getMemoryById(
+                            memId
+                        );
+        if (!existingResponse) {
+            console.log("create new memory", memId);
+            const memory = await mm.addEmbeddingToMemory({
+                id: memId,
+                agentId: runtime.agentId,
+                userId: runtime.agentId, //|| runtime.userId,
+                content: { text: prompt },
+                roomId: runtime.agentId,
+                embedding: await embed(runtime, prompt),
+            });
+            await mm.createMemory(memory);
+            return "Success";
+        }
+        else {
+            console.log("existingResponse", memId, existingResponse);
+            return "Success";
+        } 
+        
     } catch (error: any) {
         console.error("Failed to process chunk:", error);
         return "Failed";
@@ -353,18 +380,6 @@ async function processChunk(prompt: string, manager: string, runtime: AgentRunti
 
 
 
-function contentToUuid(content:string):UUID {
-    // Step 1: Create a SHA-1 hash of the content
-    const hash = crypto.createHash('sha1')
-                      .update(content)
-                      .digest('hex'); // Outputs a 40-character hex string
-
-    // Step 2: Format the hash into a UUID-like structure
-    // UUID format: 8-4-4-4-12 (e.g., 550e8400-e29b-41d4-a716-446655440000)
-    const uuid = `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20, 32)}`;
-
-    return uuid as `${string}-${string}-${string}-${string}-${string}`;
-}
 
 // async function  process_text(this: any, agentId: string, userName: string, name: string, sUserId: string | undefined,
 //     sRoomId: string, text:string, agents: Map<string, any>) {