Merge pull request #2351 from augchan42/feature/rag-knowledge-improvements-v2

feat: (ragKnowledge) Enhance RAG knowledge handling

Author: odilitime

packages/adapter-sqlite/src/index.ts

@@ -16,6 +16,7 @@ import {
     type Relationship,
     type UUID,
     RAGKnowledgeItem,
+    type ChunkRow,
 } from "@elizaos/core";
 import { Database } from "better-sqlite3";
 import { v4 } from "uuid";
@@ -966,8 +967,106 @@ export class SqliteDatabaseAdapter
     }
 
     async removeKnowledge(id: UUID): Promise<void> {
-        const sql = `DELETE FROM knowledge WHERE id = ?`;
-        this.db.prepare(sql).run(id);
+        if (typeof id !== "string") {
+            throw new Error("Knowledge ID must be a string");
+        }
+
+        try {
+            // Execute the transaction and ensure it's called with ()
+            await this.db.transaction(() => {
+                if (id.includes("*")) {
+                    const pattern = id.replace("*", "%");
+                    const sql = "DELETE FROM knowledge WHERE id LIKE ?";
+                    elizaLogger.debug(
+                        `[Knowledge Remove] Executing SQL: ${sql} with pattern: ${pattern}`
+                    );
+                    const stmt = this.db.prepare(sql);
+                    const result = stmt.run(pattern);
+                    elizaLogger.debug(
+                        `[Knowledge Remove] Pattern deletion affected ${result.changes} rows`
+                    );
+                    return result.changes; // Return changes for logging
+                } else {
+                    // Log queries before execution
+                    const selectSql = "SELECT id FROM knowledge WHERE id = ?";
+                    const chunkSql =
+                        "SELECT id FROM knowledge WHERE json_extract(content, '$.metadata.originalId') = ?";
+                    elizaLogger.debug(`[Knowledge Remove] Checking existence with:
+                        Main: ${selectSql} [${id}]
+                        Chunks: ${chunkSql} [${id}]`);
+
+                    const mainEntry = this.db.prepare(selectSql).get(id) as
+                        | ChunkRow
+                        | undefined;
+                    const chunks = this.db
+                        .prepare(chunkSql)
+                        .all(id) as ChunkRow[];
+
+                    elizaLogger.debug(`[Knowledge Remove] Found:`, {
+                        mainEntryExists: !!mainEntry?.id,
+                        chunkCount: chunks.length,
+                        chunkIds: chunks.map((c) => c.id),
+                    });
+
+                    // Execute and log chunk deletion
+                    const chunkDeleteSql =
+                        "DELETE FROM knowledge WHERE json_extract(content, '$.metadata.originalId') = ?";
+                    elizaLogger.debug(
+                        `[Knowledge Remove] Executing chunk deletion: ${chunkDeleteSql} [${id}]`
+                    );
+                    const chunkResult = this.db.prepare(chunkDeleteSql).run(id);
+                    elizaLogger.debug(
+                        `[Knowledge Remove] Chunk deletion affected ${chunkResult.changes} rows`
+                    );
+
+                    // Execute and log main entry deletion
+                    const mainDeleteSql = "DELETE FROM knowledge WHERE id = ?";
+                    elizaLogger.debug(
+                        `[Knowledge Remove] Executing main deletion: ${mainDeleteSql} [${id}]`
+                    );
+                    const mainResult = this.db.prepare(mainDeleteSql).run(id);
+                    elizaLogger.debug(
+                        `[Knowledge Remove] Main deletion affected ${mainResult.changes} rows`
+                    );
+
+                    const totalChanges =
+                        chunkResult.changes + mainResult.changes;
+                    elizaLogger.debug(
+                        `[Knowledge Remove] Total rows affected: ${totalChanges}`
+                    );
+
+                    // Verify deletion
+                    const verifyMain = this.db.prepare(selectSql).get(id);
+                    const verifyChunks = this.db.prepare(chunkSql).all(id);
+                    elizaLogger.debug(
+                        `[Knowledge Remove] Post-deletion check:`,
+                        {
+                            mainStillExists: !!verifyMain,
+                            remainingChunks: verifyChunks.length,
+                        }
+                    );
+
+                    return totalChanges; // Return changes for logging
+                }
+            })(); // Important: Call the transaction function
+
+            elizaLogger.debug(
+                `[Knowledge Remove] Transaction completed for id: ${id}`
+            );
+        } catch (error) {
+            elizaLogger.error("[Knowledge Remove] Error:", {
+                id,
+                error:
+                    error instanceof Error
+                        ? {
+                              message: error.message,
+                              stack: error.stack,
+                              name: error.name,
+                          }
+                        : error,
+            });
+            throw error;
+        }
     }
 
     async clearKnowledge(agentId: UUID, shared?: boolean): Promise<void> {

packages/core/src/environment.ts

@@ -77,15 +77,23 @@ export const CharacterSchema = z.object({
     postExamples: z.array(z.string()),
     topics: z.array(z.string()),
     adjectives: z.array(z.string()),
-    knowledge: z.array(
-        z.union([
-            z.string(),
-            z.object({
-                path: z.string(),
-                shared: z.boolean().optional()
-            })
-        ])
-    ).optional(),
+    knowledge: z
+        .array(
+            z.union([
+                z.string(), // Direct knowledge strings
+                z.object({
+                    // Individual file config
+                    path: z.string(),
+                    shared: z.boolean().optional(),
+                }),
+                z.object({
+                    // Directory config
+                    directory: z.string(),
+                    shared: z.boolean().optional(),
+                }),
+            ])
+        )
+        .optional(),
     clients: z.array(z.nativeEnum(Clients)),
     plugins: z.union([z.array(z.string()), z.array(PluginSchema)]),
     settings: z