From 277ed4bdbae3f202614bf1cdbc26e4d3553d949f Mon Sep 17 00:00:00 2001 From: Varkrishin Date: Sat, 23 Nov 2024 15:20:21 +0530 Subject: [PATCH 1/2] feat: loading characters from db at load and runtime --- agent/src/index.ts | 196 +++++++++++++++ docs/docs/guides/template-configuration.md | 22 +- packages/adapter-postgres/schema.sql | 9 + packages/adapter-postgres/src/index.ts | 61 ++++- packages/adapter-sqlite/src/index.ts | 56 +++++ packages/adapter-sqlite/src/sqliteTables.ts | 10 + packages/client-direct/src/index.ts | 63 +++++ packages/core/src/crypt.ts | 63 +++++ packages/core/src/embedding.ts | 10 +- packages/core/src/index.ts | 1 + packages/core/src/types.ts | 20 ++ scripts/importCharactersInDB/crypt.js | 112 +++++++++ .../postgres/FetchFromDb.js | 143 +++++++++++ .../postgres/insertInDb.js | 223 ++++++++++++++++++ .../sqlite/fetchFromDb.js | 111 +++++++++ .../importCharactersInDB/sqlite/insertInDb.js | 188 +++++++++++++++ 16 files changed, 1270 insertions(+), 18 deletions(-) create mode 100644 packages/core/src/crypt.ts create mode 100644 scripts/importCharactersInDB/crypt.js create mode 100644 scripts/importCharactersInDB/postgres/FetchFromDb.js create mode 100644 scripts/importCharactersInDB/postgres/insertInDb.js create mode 100644 scripts/importCharactersInDB/sqlite/fetchFromDb.js create mode 100644 scripts/importCharactersInDB/sqlite/insertInDb.js diff --git a/agent/src/index.ts b/agent/src/index.ts index cb580eed776..c6814959535 100644 --- a/agent/src/index.ts +++ b/agent/src/index.ts @@ -33,6 +33,13 @@ import path from "path"; import { fileURLToPath } from "url"; import { character } from "./character.ts"; import type { DirectClient } from "@ai16z/client-direct"; +import { Pool } from "pg"; +import { EncryptionUtil } from "@ai16z/eliza"; +import express, { Request as ExpressRequest } from "express"; +import bodyParser from "body-parser"; +import cors from "cors"; + +let globalDirectClient: DirectClient | null = null; const __filename = fileURLToPath(import.meta.url); // get the resolved path to the file const __dirname = path.dirname(__filename); // get the name of the directory @@ -306,8 +313,17 @@ async function startAgent(character: Character, directClient: DirectClient) { } } +export function setGlobalDirectClient(client: DirectClient) { + globalDirectClient = client; +} + +export function getGlobalDirectClient(): DirectClient | null { + return globalDirectClient; +} + const startAgents = async () => { const directClient = await DirectClientInterface.start(); + setGlobalDirectClient(directClient as DirectClient); const args = parseArguments(); let charactersArg = args.characters || args.character; @@ -318,6 +334,15 @@ const startAgents = async () => { characters = await loadCharacters(charactersArg); } + const shouldFetchFromDb = process.env.FETCH_FROM_DB === "true"; + + if (shouldFetchFromDb) { + characters = await loadCharactersFromDb(charactersArg); + if (characters.length === 0) { + characters = [character]; + } + } + try { for (const character of characters) { await startAgent(character, directClient as DirectClient); @@ -384,3 +409,174 @@ async function handleUserInput(input, agentId) { console.error("Error fetching response:", error); } } + +/** + * Loads characters from PostgreSQL database + * @param characterNames - Optional comma-separated list of character names to load + * @returns Promise of loaded and decrypted characters + */ +export async function loadCharactersFromDb( + characterNames?: string +): Promise { + try { + const encryptionUtil = new EncryptionUtil( + process.env.ENCRYPTION_KEY || "default-key" + ); + + const dataDir = path.join(__dirname, "../data"); + + if (!fs.existsSync(dataDir)) { + fs.mkdirSync(dataDir, { recursive: true }); + } + + const db = initializeDatabase(dataDir); + await db.init(); + + // Convert names to UUIDs if provided + const characterIds = characterNames + ?.split(",") + .map((name) => name.trim()) + .map((name) => stringToUuid(name)); + + // Get characters and their secretsIVs from database + const [characters, secretsIVs] = await db.loadCharacters(characterIds); + + if (characters.length === 0) { + elizaLogger.log( + "No characters found in database, using default character" + ); + return []; + } + + // Process each character with its corresponding secretsIV + const processedCharacters = await Promise.all( + characters.map(async (character, index) => { + try { + // Decrypt secrets if they exist + if (character.settings?.secrets) { + const decryptedSecrets: { [key: string]: string } = {}; + const secretsIV = secretsIVs[index]; + + for (const [key, encryptedValue] of Object.entries( + character.settings.secrets + )) { + const iv = secretsIV[key]; + if (!iv) { + elizaLogger.error( + `Missing IV for secret ${key} in character ${character.name}` + ); + continue; + } + + try { + decryptedSecrets[key] = encryptionUtil.decrypt({ + encryptedText: encryptedValue, + iv, + }); + } catch (error) { + elizaLogger.error( + `Failed to decrypt secret ${key} for character ${character.name}:`, + error + ); + } + } + character.settings.secrets = decryptedSecrets; + } + + // Handle plugins + if (character.plugins) { + elizaLogger.log("Plugins are: ", character.plugins); + const importedPlugins = await Promise.all( + character.plugins.map(async (plugin) => { + // if the plugin name doesnt start with @eliza, + + const importedPlugin = await import( + plugin.name + ); + return importedPlugin; + }) + ); + + character.plugins = importedPlugins; + } + + validateCharacterConfig(character); + elizaLogger.log( + `Character loaded from db: ${character.name}` + ); + console.log("-------------------------------"); + return character; + } catch (error) { + elizaLogger.error( + `Error processing character ${character.name}:`, + error + ); + throw error; + } + }) + ); + + return processedCharacters; + } catch (error) { + elizaLogger.error("Database error:", error); + elizaLogger.log("Falling back to default character"); + return [defaultCharacter]; + } +} + +// If dynamic loading is enabled, start the express server +// we can directly call this endpoint to load an agent +// otherwise we can use the direct client as a proxy if we +// want to expose only single post to public +if (process.env.AGENT_RUNTIME_MANAGEMENT === "true") { + const app = express(); + app.use(cors()); + app.use(bodyParser.json()); + + // This endpoint can be directly called or + app.post( + "/load/:agentName", + async (req: ExpressRequest, res: express.Response) => { + try { + const agentName = req.params.agentName; + const characters = await loadCharactersFromDb(agentName); + + if (characters.length === 0) { + res.status(404).json({ + success: false, + error: `Character ${agentName} does not exist in DB`, + }); + return; + } + + const directClient = getGlobalDirectClient(); + await startAgent(characters[0], directClient); + + res.json({ + success: true, + port: settings.SERVER_PORT, + character: { + id: characters[0].id, + name: characters[0].name, + }, + }); + } catch (error) { + elizaLogger.error(`Error loading agent:`, error); + res.status(500).json({ + success: false, + error: error.message, + }); + } + } + ); + + const agentPort = settings.AGENT_PORT + ? parseInt(settings.AGENT_PORT) + : 3001; + //if agent port is 0, it means we want to use a random port + const server = app.listen(agentPort, () => { + elizaLogger.success( + `Agent server running at http://localhost:${agentPort}/` + ); + }); +} diff --git a/docs/docs/guides/template-configuration.md b/docs/docs/guides/template-configuration.md index febeb02f4fc..4e5376b6714 100644 --- a/docs/docs/guides/template-configuration.md +++ b/docs/docs/guides/template-configuration.md @@ -15,14 +15,14 @@ Here are all the template options you can configure: ```json { "templates": { - "goalsTemplate": "", // Define character goals - "factsTemplate": "", // Specify character knowledge - "messageHandlerTemplate": "", // Handle general messages - "shouldRespondTemplate": "", // Control response triggers + "goalsTemplate": "", // Define character goals + "factsTemplate": "", // Specify character knowledge + "messageHandlerTemplate": "", // Handle general messages + "shouldRespondTemplate": "", // Control response triggers "continueMessageHandlerTemplate": "", // Manage conversation flow - "evaluationTemplate": "", // Handle response evaluation - "twitterSearchTemplate": "", // Process Twitter searches - "twitterPostTemplate": "", // Format Twitter posts + "evaluationTemplate": "", // Handle response evaluation + "twitterSearchTemplate": "", // Process Twitter searches + "twitterPostTemplate": "", // Format Twitter posts "twitterMessageHandlerTemplate": "", // Handle Twitter messages "twitterShouldRespondTemplate": "", // Control Twitter responses "telegramMessageHandlerTemplate": "", // Handle Telegram messages @@ -60,11 +60,11 @@ Configure platform-specific behaviors for your character, such as handling direc "clientConfig": { "telegram": { "shouldIgnoreDirectMessages": true, // Ignore DMs - "shouldIgnoreBotMessages": true // Ignore bot messages + "shouldIgnoreBotMessages": true // Ignore bot messages }, "discord": { - "shouldIgnoreBotMessages": true, // Ignore bot messages - "shouldIgnoreDirectMessages": true // Ignore DMs + "shouldIgnoreBotMessages": true, // Ignore bot messages + "shouldIgnoreDirectMessages": true // Ignore DMs } } } @@ -73,11 +73,13 @@ Configure platform-specific behaviors for your character, such as handling direc ## Best Practices 1. **Template Management** + - Keep templates focused and specific - Use clear, consistent formatting - Document custom template behavior 2. **Client Configuration** + - Configure per platform as needed - Test behavior in development - Monitor interaction patterns diff --git a/packages/adapter-postgres/schema.sql b/packages/adapter-postgres/schema.sql index e1122136c12..74aa8a06394 100644 --- a/packages/adapter-postgres/schema.sql +++ b/packages/adapter-postgres/schema.sql @@ -26,6 +26,15 @@ CREATE TABLE IF NOT EXISTS accounts ( "details" JSONB DEFAULT '{}'::jsonb ); +CREATE TABLE IF NOT EXISTS characters ( + "id" UUID PRIMARY KEY, + "name" TEXT, + "characterState" JSONB NOT NULL, + "secretsIV" JSONB DEFAULT '{}'::jsonb, + "createdAt" TIMESTAMP DEFAULT now(), + "updatedAt" TIMESTAMP DEFAULT now() +); + CREATE TABLE IF NOT EXISTS rooms ( "id" UUID PRIMARY KEY, "createdAt" TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP diff --git a/packages/adapter-postgres/src/index.ts b/packages/adapter-postgres/src/index.ts index 23a4b426ae9..11726d863df 100644 --- a/packages/adapter-postgres/src/index.ts +++ b/packages/adapter-postgres/src/index.ts @@ -9,6 +9,10 @@ import { type Relationship, type UUID, type IDatabaseCacheAdapter, + type IDatabaseAdapter, + type CharacterTable, + type Secrets, + type Character, Participant, DatabaseAdapter, elizaLogger, @@ -22,7 +26,7 @@ const __dirname = path.dirname(__filename); // get the name of the directory export class PostgresDatabaseAdapter extends DatabaseAdapter - implements IDatabaseCacheAdapter + implements IDatabaseCacheAdapter, IDatabaseAdapter { private pool: Pool; @@ -39,6 +43,7 @@ export class PostgresDatabaseAdapter ...defaultConfig, ...connectionConfig, // Allow overriding defaults }); + this.db = this.pool; this.pool.on("error", async (err) => { elizaLogger.error("Unexpected error on idle client", err); @@ -736,7 +741,9 @@ export class PostgresDatabaseAdapter ); if (existingParticipant.rows.length > 0) { - console.log(`Participant with userId ${userId} already exists in room ${roomId}.`); + console.log( + `Participant with userId ${userId} already exists in room ${roomId}.` + ); return; // Exit early if the participant already exists } @@ -750,11 +757,13 @@ export class PostgresDatabaseAdapter } catch (error) { // This is to prevent duplicate participant error in case of a race condition // Handle unique constraint violation error (code 23505) - if (error.code === '23505') { - console.warn(`Participant with userId ${userId} already exists in room ${roomId}.`); // Optionally, you can log this or handle it differently + if (error.code === "23505") { + console.warn( + `Participant with userId ${userId} already exists in room ${roomId}.` + ); // Optionally, you can log this or handle it differently } else { // Handle other errors - console.error('Error adding participant:', error); + console.error("Error adding participant:", error); return false; } } finally { @@ -958,6 +967,48 @@ export class PostgresDatabaseAdapter client.release(); } } + + /** + * Loads characters from database + * @param characterIds Optional array of character UUIDs to load + * @returns Promise of tuple containing Characters array and their corresponding SecretsIV + */ + async loadCharacters( + characterIds?: UUID[] + ): Promise<[Character[], Secrets[]]> { + const client = await this.pool.connect(); + try { + let query = + 'SELECT "id", "name", "characterState", "secretsIV" FROM characters'; + const queryParams: any[] = []; + + if (characterIds?.length) { + query += ' WHERE "id" = ANY($1)'; + queryParams.push(characterIds); + } + + query += " ORDER BY name"; + const result = await client.query( + query, + queryParams + ); + + const characters: Character[] = []; + const secretsIVs: Secrets[] = []; + + for (const row of result.rows) { + characters.push(row.characterState); + secretsIVs.push(row.secretsIV || {}); + } + + return [characters, secretsIVs]; + } catch (error) { + elizaLogger.error("Error loading characters:", error); + throw error; + } finally { + client.release(); + } + } } export default PostgresDatabaseAdapter; diff --git a/packages/adapter-sqlite/src/index.ts b/packages/adapter-sqlite/src/index.ts index 52f1ac59797..6b88f43cc3b 100644 --- a/packages/adapter-sqlite/src/index.ts +++ b/packages/adapter-sqlite/src/index.ts @@ -12,6 +12,9 @@ import { type Memory, type Relationship, type UUID, + type CharacterTable, + type Secrets, + type Character, } from "@ai16z/eliza"; import { Database } from "better-sqlite3"; import { v4 } from "uuid"; @@ -708,4 +711,57 @@ export class SqliteDatabaseAdapter return false; } } + + /** + * Loads characters from database + * @param characterIds Optional array of character UUIDs to load + * @returns Promise of tuple containing Characters array and their corresponding SecretsIV + */ + async loadCharacters( + characterIds?: UUID[] + ): Promise<[Character[], Secrets[]]> { + try { + let sql = + "SELECT id, name, characterState, secretsIV FROM characters"; + let queryParams: any[] = []; + + if (characterIds?.length) { + // Create placeholders for the IN clause + const placeholders = characterIds.map(() => "?").join(","); + sql += ` WHERE id IN (${placeholders})`; + queryParams = characterIds; + } + + sql += " ORDER BY name"; + + // SQLite returns JSON as string, so we need to parse it + const stmt = this.db.prepare(sql); + const rows = stmt.all(...queryParams) as CharacterTable[]; + + const characters: Character[] = []; + const secretsIVs: Secrets[] = []; + + for (const row of rows) { + // Parse characterState if it's a string (SQLite stores JSON as text) + const characterState = + typeof row.characterState === "string" + ? JSON.parse(row.characterState) + : row.characterState; + + // Parse secretsIV if it's a string + const secretsIV = + typeof row.secretsIV === "string" + ? JSON.parse(row.secretsIV) + : row.secretsIV || {}; + + characters.push(characterState); + secretsIVs.push(secretsIV); + } + + return [characters, secretsIVs]; + } catch (error) { + console.error("Error loading characters:", error); + throw error; + } + } } diff --git a/packages/adapter-sqlite/src/sqliteTables.ts b/packages/adapter-sqlite/src/sqliteTables.ts index fdd47e5697f..c9b3f825426 100644 --- a/packages/adapter-sqlite/src/sqliteTables.ts +++ b/packages/adapter-sqlite/src/sqliteTables.ts @@ -13,6 +13,16 @@ CREATE TABLE IF NOT EXISTS "accounts" ( "details" TEXT DEFAULT '{}' CHECK(json_valid("details")) -- Ensuring details is a valid JSON field ); +-- Table: characters +CREATE TABLE IF NOT EXISTS characters ( + "id" TEXT PRIMARY KEY, -- SQLite does not have a UUID type; use TEXT for UUID + "name" TEXT, + "characterState" TEXT NOT NULL, -- SQLite does not have JSONB; use TEXT to store JSON + "secretsIV" TEXT DEFAULT '{}', -- Store JSON as TEXT and provide default value + "createdAt" DATETIME DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME DEFAULT CURRENT_TIMESTAMP +); + -- Table: memories CREATE TABLE IF NOT EXISTS "memories" ( "id" TEXT PRIMARY KEY, diff --git a/packages/client-direct/src/index.ts b/packages/client-direct/src/index.ts index 123600bf555..61f18b36a1b 100644 --- a/packages/client-direct/src/index.ts +++ b/packages/client-direct/src/index.ts @@ -267,6 +267,69 @@ export class DirectClient { res.json({ images: imagesRes }); } ); + + this.app.post( + "/load/:agentName", // name as its easier to remember and + //id is an uuid derived from name + async (req: express.Request, res: express.Response) => { + try { + if (process.env.AGENT_RUNTIME_MANAGEMENT !== "true") { + throw new Error( + "Agent runtime management is not enabled" + ); + } + + const agentName = req.params.agentName; + const agentId = stringToUuid(agentName); + // Check if agent is already running + let runtime = this.agents.get(agentId); + if (runtime) { + res.status(409).json({ + success: false, + error: `Agent ${agentName} already running`, + }); + return; + } + const agentPort = process.env.AGENT_PORT + ? parseInt(process.env.AGENT_PORT) + : 3001; + + // Forward request to agent server + const response = await fetch( + `http://localhost:${agentPort}/load/${agentName}`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + } + ); + if (!response.ok) { + const errorData = await response.json(); + // Process the error from the agent + res.status(response.status).json({ + success: false, + error: + errorData.error || + "Unknown error occurred while loading the agent", + }); + return; // Exit the function after handling the error + } + const data = await response.json(); + res.json(data); + } catch (error) { + console.error("Error loading agent:", error); + res.status(500).json({ + success: false, + error: `Error loading agent ${error}`, + }); + } + } + ); + } + + public getAgent(agentId: string) { + return this.agents.get(agentId); } public registerAgent(runtime: AgentRuntime) { diff --git a/packages/core/src/crypt.ts b/packages/core/src/crypt.ts new file mode 100644 index 00000000000..41e8e2d7dcc --- /dev/null +++ b/packages/core/src/crypt.ts @@ -0,0 +1,63 @@ +import crypto from "crypto"; + +export interface EncryptedData { + encryptedText: string; + iv: string; +} + +export class EncryptionUtil { + private readonly algorithm = "aes-256-cbc"; + private readonly key: Buffer; + + constructor(secretKey: string) { + // Create a 32-byte key using SHA-512 hash of the secret key + this.key = Buffer.from( + crypto + .createHash("sha512") + .update(secretKey) + .digest("hex") + .substring(0, 32), + "utf8" + ); + } + + encrypt(data: string): EncryptedData { + // Generate a random IV for each encryption + const iv = crypto.randomBytes(16); + + // Create cipher with key and iv + const cipher = crypto.createCipheriv(this.algorithm, this.key, iv); + + // Encrypt the data + let encrypted = cipher.update(data, "utf8", "hex"); + encrypted += cipher.final("hex"); + + // Return both the encrypted data and iv + return { + encryptedText: encrypted, + iv: iv.toString("hex"), + }; + } + + decrypt(data: EncryptedData): string { + try { + // Convert IV back to Buffer + const iv = Buffer.from(data.iv, "hex"); + + // Create decipher + const decipher = crypto.createDecipheriv( + this.algorithm, + this.key, + iv + ); + + // Decrypt the data + let decrypted = decipher.update(data.encryptedText, "hex", "utf8"); + decrypted += decipher.final("utf8"); + + return decrypted; + } catch (error) { + throw new Error("Decryption failed: " + (error as Error).message); + } + } +} diff --git a/packages/core/src/embedding.ts b/packages/core/src/embedding.ts index c14f8b38ce6..030692ac712 100644 --- a/packages/core/src/embedding.ts +++ b/packages/core/src/embedding.ts @@ -160,9 +160,11 @@ async function getLocalEmbedding(input: string): Promise { return await import("fastembed"); } catch (error) { elizaLogger.error("Failed to load fastembed."); - throw new Error("fastembed import failed, falling back to remote embedding"); + throw new Error( + "fastembed import failed, falling back to remote embedding" + ); } - })() + })(), ]); const [fs, { fileURLToPath }, fastEmbed] = moduleImports; @@ -194,7 +196,9 @@ async function getLocalEmbedding(input: string): Promise { const embedding = await embeddingModel.queryEmbed(trimmedInput); return embedding; } catch (error) { - elizaLogger.warn("Local embedding not supported in browser, falling back to remote embedding."); + elizaLogger.warn( + "Local embedding not supported in browser, falling back to remote embedding." + ); throw new Error("Local embedding not supported in browser"); } } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 96877411cc3..8eb8f5c09b0 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -20,4 +20,5 @@ export * from "./parsing.ts"; export * from "./uuid.ts"; export * from "./enviroment.ts"; export * from "./cache.ts"; +export * from "./crypt.ts"; export { default as knowledge } from "./knowledge.ts"; diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index f6ceb828679..7991126b4d0 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -493,6 +493,19 @@ export interface Account { avatarUrl?: string; } +/** + * Represents a character in db + * Stored Character as JSONB as the characterState is more efficient for jsonb/nosql + * Other approach is to store each field as a table like character_settings, + * character_knowledge, character_plugins, etc. + */ +export type CharacterTable = { + id: UUID; + name: string; + characterState: Character; + secretsIV?: Secrets; +}; + /** * Room participant with account details */ @@ -584,6 +597,13 @@ export enum Clients { TWITTER = "twitter", TELEGRAM = "telegram", } +/** + * Configuration for an agent secrets + */ +export type Secrets = { + [key: string]: string; +}; + /** * Configuration for an agent character */ diff --git a/scripts/importCharactersInDB/crypt.js b/scripts/importCharactersInDB/crypt.js new file mode 100644 index 00000000000..23db4d23ff4 --- /dev/null +++ b/scripts/importCharactersInDB/crypt.js @@ -0,0 +1,112 @@ +// js version of the EncryptionUtil class in core/src/crypt.ts +// also added stringToUuid function +const crypto = require("crypto"); +const sha1 = require("js-sha1"); + +class EncryptionUtil { + constructor(secretKey) { + this.algorithm = "aes-256-cbc"; + // Create a 32-byte key using SHA-512 hash of the secret key + this.key = Buffer.from( + crypto + .createHash("sha512") + .update(secretKey) + .digest("hex") + .substring(0, 32), + "utf8" + ); + } + + encrypt(data) { + // Generate a random IV for each encryption + const iv = crypto.randomBytes(16); + + // Create cipher with key and iv + const cipher = crypto.createCipheriv(this.algorithm, this.key, iv); + + // Encrypt the data + let encrypted = cipher.update(data, "utf8", "hex"); + encrypted += cipher.final("hex"); + + // Return both the encrypted data and iv + return { + encryptedText: encrypted, + iv: iv.toString("hex"), + }; + } + + decrypt(data) { + try { + // Convert IV back to Buffer + const iv = Buffer.from(data.iv, "hex"); + + // Create decipher + const decipher = crypto.createDecipheriv( + this.algorithm, + this.key, + iv + ); + + // Decrypt the data + let decrypted = decipher.update(data.encryptedText, "hex", "utf8"); + decrypted += decipher.final("utf8"); + + return decrypted; + } catch (error) { + throw new Error("Decryption failed: " + error.message); + } + } + + stringToUuid(target) { + if (typeof target === "number") { + target = target.toString(); + } + + if (typeof target !== "string") { + throw TypeError("Value must be string"); + } + + const _uint8ToHex = (ubyte) => { + const first = ubyte >> 4; + const second = ubyte - (first << 4); + const HEX_DIGITS = "0123456789abcdef".split(""); + return HEX_DIGITS[first] + HEX_DIGITS[second]; + }; + + const _uint8ArrayToHex = (buf) => { + let out = ""; + for (let i = 0; i < buf.length; i++) { + out += _uint8ToHex(buf[i]); + } + return out; + }; + + const escapedStr = encodeURIComponent(target); + const buffer = new Uint8Array(escapedStr.length); + for (let i = 0; i < escapedStr.length; i++) { + buffer[i] = escapedStr[i].charCodeAt(0); + } + + const hash = sha1(buffer); + const hashBuffer = new Uint8Array(hash.length / 2); + for (let i = 0; i < hash.length; i += 2) { + hashBuffer[i / 2] = parseInt(hash.slice(i, i + 2), 16); + } + + return ( + _uint8ArrayToHex(hashBuffer.slice(0, 4)) + + "-" + + _uint8ArrayToHex(hashBuffer.slice(4, 6)) + + "-" + + _uint8ToHex(hashBuffer[6] & 0x0f) + + _uint8ToHex(hashBuffer[7]) + + "-" + + _uint8ToHex((hashBuffer[8] & 0x3f) | 0x80) + + _uint8ToHex(hashBuffer[9]) + + "-" + + _uint8ArrayToHex(hashBuffer.slice(10, 16)) + ); + } +} + +module.exports = EncryptionUtil; diff --git a/scripts/importCharactersInDB/postgres/FetchFromDb.js b/scripts/importCharactersInDB/postgres/FetchFromDb.js new file mode 100644 index 00000000000..131dd682a92 --- /dev/null +++ b/scripts/importCharactersInDB/postgres/FetchFromDb.js @@ -0,0 +1,143 @@ +/** + * Character Database Fetch Script + * + * This script retrieves and displays character data from PostgreSQL database. + * Features: + * - Fetches all characters + * - Decrypts stored secrets using separate IVs + * - Pretty prints character data and decrypted secrets + * - Displays creation and update timestamps + * + * Usage: + * Requires environment variables: + * - POSTGRES_URL: PostgreSQL connection string + * - ENCRYPTION_KEY: Key for decrypting secrets + */ + +const { Pool } = require("pg"); +const EncryptionUtil = require("../crypt"); +require("dotenv").config(); + +// Database connection pool +const pool = new Pool({ + connectionString: process.env.POSTGRES_URL, +}); + +// Encryption utility instance +const encryptionUtil = new EncryptionUtil( + process.env.ENCRYPTION_KEY || "default-key" +); + +/** + * Tests database connection + * Exits process if connection fails + */ +async function testConnection() { + const client = await pool.connect(); + try { + await client.query("SELECT NOW()"); + console.log("Database connection successful"); + } catch (error) { + console.error("Failed to connect to database:", error.message); + process.exit(1); + } finally { + client.release(); + } +} + +/** + * Decrypts character secrets + * @param {Object} encryptedSecrets - Encrypted secrets from characterState + * @param {Object} secretsIV - IVs for each secret + * @returns {Object} Decrypted secrets + */ +async function decryptSecrets(encryptedSecrets, secretsIV) { + if (!encryptedSecrets || !secretsIV) return {}; + + const decryptedSecrets = {}; + for (const [key, encryptedValue] of Object.entries(encryptedSecrets)) { + try { + const iv = secretsIV[key]; + if (!iv) { + console.error(`Missing IV for secret ${key}`); + continue; + } + + const decrypted = encryptionUtil.decrypt({ + encryptedText: encryptedValue, + iv: iv, + }); + decryptedSecrets[key] = decrypted; + } catch (error) { + console.error(`Failed to decrypt secret ${key}:`, error.message); + } + } + return decryptedSecrets; +} + +/** + * Fetches and displays all characters + * Handles: + * - Database query + * - Pretty printing of character data + * - Secret decryption and display + * - Error handling + */ +async function fetchCharacters() { + const client = await pool.connect(); + try { + const result = await client.query( + 'SELECT * FROM characters ORDER BY "name"' + ); + + for (const row of result.rows) { + console.log("\n=== Character ==="); + console.log("ID:", row.id); + console.log("Name:", row.name); + + // Create a copy of character state for display + const displayState = JSON.parse(JSON.stringify(row.characterState)); + + // Decrypt secrets if they exist + if ( + displayState.settings?.secrets && + Object.keys(displayState.settings.secrets).length > 0 + ) { + const decryptedSecrets = await decryptSecrets( + displayState.settings.secrets, + row.secretsIV + ); + // Replace encrypted secrets with decrypted ones for display + displayState.settings.secrets = decryptedSecrets; + } + + // Pretty print character state with decrypted secrets + console.log("\nCharacter State:"); + console.log(JSON.stringify(displayState, null, 2)); + + console.log("\nCreated At:", row.createdAt); + console.log("Updated At:", row.updatedAt); + console.log("==================\n"); + } + + console.log(`Total characters: ${result.rows.length}`); + } catch (error) { + console.error("Error fetching characters:", error); + } finally { + client.release(); + } +} + +// Main execution +testConnection() + .then(async () => { + console.log("Starting fetch operation..."); + await fetchCharacters(); + }) + .catch((error) => { + console.error("Failed to process:", error); + process.exit(1); + }) + .finally(() => { + pool.end(); + }); diff --git a/scripts/importCharactersInDB/postgres/insertInDb.js b/scripts/importCharactersInDB/postgres/insertInDb.js new file mode 100644 index 00000000000..13e20838fd7 --- /dev/null +++ b/scripts/importCharactersInDB/postgres/insertInDb.js @@ -0,0 +1,223 @@ +/** + * Character Database Insertion Script + * + * This script reads character JSON files and inserts them into a PostgreSQL database. + * It handles both single files and directories of JSON files. + * Features: + * - Validates JSON against a Zod schema + * - Encrypts sensitive data (secrets) before storage + * - Generates deterministic UUIDs from character names + * - Stores character state and encrypted secrets separately + * - Supports upsert operations (update if exists) + * + * Usage: + * Requires environment variables: + * - POSTGRES_URL: PostgreSQL connection string + * - ENCRYPTION_KEY: Key for encrypting secrets + * - INPUT_PATH: Path to JSON file or directory + */ + +const fs = require("fs").promises; +const path = require("path"); +const { z } = require("zod"); +const { Pool } = require("pg"); +const EncryptionUtil = require("../crypt"); +require("dotenv").config(); + +// Zod schema for validating character JSON structure +const CharacterSchema = z.object({ + name: z.string(), + username: z.string().optional(), + system: z.string().optional(), + modelProvider: z.string(), + modelEndpointOverride: z.string().optional(), + bio: z.union([z.string(), z.array(z.string())]), + lore: z.array(z.string()), + messageExamples: z.array(z.array(z.any())), + postExamples: z.array(z.string()), + people: z.array(z.string()), + topics: z.array(z.string()), + adjectives: z.array(z.string()), + knowledge: z.array(z.string()).optional(), + clients: z.array(z.string()), + plugins: z.array(z.string()), + settings: z + .object({ + secrets: z.record(z.string()).optional(), + voice: z + .object({ + model: z.string().optional(), + url: z.string().optional(), + }) + .optional(), + }) + .optional(), + style: z.object({ + all: z.array(z.string()), + chat: z.array(z.string()), + post: z.array(z.string()), + }), +}); + +// Database connection pool +const pool = new Pool({ + connectionString: process.env.POSTGRES_URL, +}); + +// Encryption utility instance +const encryptionUtil = new EncryptionUtil( + process.env.ENCRYPTION_KEY || "default-key" +); + +/** + * Tests database connection + * Exits process if connection fails + */ +async function testConnection() { + const client = await pool.connect(); + try { + await client.query("SELECT NOW()"); + console.log("Database connection successful"); + } catch (error) { + console.error("Failed to connect to database:", error.message); + process.exit(1); + } finally { + client.release(); + } +} + +/** + * Inserts or updates a character in the database + * @param {Object} character - Validated character object + * Handles: + * - UUID generation + * - Secret encryption with separate IV storage + * - Character state preparation with encrypted secrets + * - Transaction management + */ +async function insertCharacter(character) { + const client = await pool.connect(); + try { + await client.query("BEGIN"); + + // Generate UUID from name + const id = encryptionUtil.stringToUuid(character.name); + + // Extract and encrypt secrets if they exist + let secretsIV = {}; + let characterState = { ...character }; + + if (character.settings?.secrets) { + const secretEntries = Object.entries(character.settings.secrets); + characterState.settings = { + ...character.settings, + secrets: {}, + }; + + for (const [key, value] of secretEntries) { + const encrypted = encryptionUtil.encrypt(value); + characterState.settings.secrets[key] = encrypted.encryptedText; + secretsIV[key] = encrypted.iv; + } + } + + const query = ` + INSERT INTO characters ( + id, + name, + "characterState", + "secretsIV", + "createdAt", + "updatedAt" + ) VALUES ($1, $2, $3, $4, NOW(), NOW()) + ON CONFLICT (id) DO UPDATE SET + "characterState" = EXCLUDED."characterState", + "secretsIV" = EXCLUDED."secretsIV", + "updatedAt" = NOW() + `; + + await client.query(query, [ + id, + character.name, + characterState, + secretsIV, + ]); + + await client.query("COMMIT"); + console.log(`Successfully imported character: ${character.name}`); + } catch (error) { + await client.query("ROLLBACK"); + console.error(`Error importing character ${character.name}:`, error); + throw error; + } finally { + client.release(); + } +} + +/** + * Processes a single JSON file + * @param {string} filePath - Path to JSON file + * Handles: + * - File reading + * - JSON parsing + * - Schema validation + * - Character insertion + */ +async function processJsonFile(filePath) { + try { + const content = await fs.readFile(filePath, "utf8"); + const character = JSON.parse(content); + + // Validate against schema + const validatedCharacter = CharacterSchema.parse(character); + + await insertCharacter(validatedCharacter); + } catch (error) { + console.error(`Error processing file ${filePath}:`, error); + } +} + +/** + * Processes input path (file or directory) + * @param {string} inputPath - Path to process + * Handles both single JSON files and directories + */ +async function processPath(inputPath) { + try { + const stats = await fs.stat(inputPath); + + if (stats.isDirectory()) { + const files = await fs.readdir(inputPath); + for (const file of files) { + if (file.endsWith(".json")) { + await processJsonFile(path.join(inputPath, file)); + } + } + } else if (stats.isFile() && inputPath.endsWith(".json")) { + await processJsonFile(inputPath); + } + } catch (error) { + console.error("Error processing path:", error); + } +} + +// Usage +const inputPath = process.env.INPUT_PATH; +if (!inputPath) { + console.error("Please provide a path to a JSON file or directory"); + process.exit(1); +} +console.log(inputPath); + +testConnection() + .then(() => { + console.log("Successful Connection"); + return processPath(inputPath); + }) + .catch((error) => { + console.error("Failed to process:", error); + process.exit(1); + }) + .finally(() => { + pool.end(); + }); diff --git a/scripts/importCharactersInDB/sqlite/fetchFromDb.js b/scripts/importCharactersInDB/sqlite/fetchFromDb.js new file mode 100644 index 00000000000..9119bd97fb9 --- /dev/null +++ b/scripts/importCharactersInDB/sqlite/fetchFromDb.js @@ -0,0 +1,111 @@ +import path from "path"; +import { fileURLToPath } from "url"; +import sqlite3 from "better-sqlite3"; // Use default export for better-sqlite3 +import EncryptionUtil from "../crypt.js"; // Ensure this file is available +import dotenv from "dotenv"; + +dotenv.config(); + +// Get __filename and __dirname in ESM +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const dataDir = path.join(__dirname, "../../../agent/data"); + +// Define SQLite database path +const dbDir = process.env.SQLITE_DB_PATH || path.join(dataDir, "db.sqlite"); + +// Initialise better-sqlite3 Database instance +const db = new sqlite3(dbDir); + +// Encryption utility instance +const encryptionUtil = new EncryptionUtil( + process.env.ENCRYPTION_KEY || "default-key" +); + +/** + * Decrypts character secrets + * @param {Object} encryptedSecrets - Encrypted secrets from characterState + * @param {Object} secretsIV - IVs for each secret + * @returns {Object} Decrypted secrets + */ +function decryptSecrets(encryptedSecrets, secretsIV) { + if (!encryptedSecrets || !secretsIV) return {}; + + const decryptedSecrets = {}; + for (const [key, encryptedValue] of Object.entries(encryptedSecrets)) { + try { + const iv = secretsIV[key]; + if (!iv) { + console.error(`Missing IV for secret ${key}`); + continue; + } + + const decrypted = encryptionUtil.decrypt({ + encryptedText: encryptedValue, + iv: iv, + }); + decryptedSecrets[key] = decrypted; + } catch (error) { + console.error(`Failed to decrypt secret ${key}:`, error.message); + } + } + return decryptedSecrets; +} + +/** + * Fetches and displays all characters + * Handles: + * - Database query + * - Pretty printing of character data + * - Secret decryption and display + * - Error handling + */ +function fetchCharacters() { + try { + const rows = db.prepare("SELECT * FROM characters ORDER BY name").all(); + + for (const row of rows) { + console.log("\n=== Character ==="); + console.log("ID:", row.id); + console.log("Name:", row.name); + + // Parse character state and secretsIV + const characterState = JSON.parse(row.characterState); + const secretsIV = JSON.parse(row.secretsIV); + + // Decrypt secrets if they exist + if ( + characterState.settings?.secrets && + Object.keys(characterState.settings.secrets).length > 0 + ) { + const decryptedSecrets = decryptSecrets( + characterState.settings.secrets, + secretsIV + ); + // Replace encrypted secrets with decrypted ones for display + characterState.settings.secrets = decryptedSecrets; + } + + // Pretty print character state with decrypted secrets + console.log("\nCharacter State:"); + console.log(JSON.stringify(characterState, null, 2)); + + console.log("\nCreated At:", row.createdAt); + console.log("Updated At:", row.updatedAt); + console.log("==================\n"); + } + + console.log(`Total characters: ${rows.length}`); + } catch (error) { + console.error("Error fetching characters:", error); + } +} + +// Main execution +try { + console.log("Starting fetch operation..."); + fetchCharacters(); +} catch (error) { + console.error("Failed to process:", error); + process.exit(1); +} diff --git a/scripts/importCharactersInDB/sqlite/insertInDb.js b/scripts/importCharactersInDB/sqlite/insertInDb.js new file mode 100644 index 00000000000..9d1ac6b2f89 --- /dev/null +++ b/scripts/importCharactersInDB/sqlite/insertInDb.js @@ -0,0 +1,188 @@ +import fs from "fs/promises"; +import path from "path"; +import { z } from "zod"; +import { fileURLToPath } from "url"; +import sqlite3 from "better-sqlite3"; // Use default export for better-sqlite3 +import EncryptionUtil from "../crypt.js"; // Ensure this file is available +import dotenv from "dotenv"; + +dotenv.config(); + +// Get __filename and __dirname in ESM +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const dataDir = path.join(__dirname, "../../../agent/data"); + +// Define SQLite database path +const dbDir = process.env.SQLITE_DB_PATH || path.join(dataDir, "db.sqlite"); + +// Ensure the database directory exists +const dbPath = path.dirname(dbDir); // Extract the directory portion of dbDir +await fs.mkdir(dbPath, { recursive: true }); // Create directory if it doesn't exist + +// Initialise better-sqlite3 Database instance +const db = new sqlite3(dbDir); + +// Zod schema for validating character JSON structure +const CharacterSchema = z.object({ + name: z.string(), + username: z.string().optional(), + system: z.string().optional(), + modelProvider: z.string(), + modelEndpointOverride: z.string().optional(), + bio: z.union([z.string(), z.array(z.string())]), + lore: z.array(z.string()), + messageExamples: z.array(z.array(z.any())), + postExamples: z.array(z.string()), + people: z.array(z.string()), + topics: z.array(z.string()), + adjectives: z.array(z.string()), + knowledge: z.array(z.string()).optional(), + clients: z.array(z.string()), + plugins: z.array(z.string()), + settings: z + .object({ + secrets: z.record(z.string()).optional(), + voice: z + .object({ + model: z.string().optional(), + url: z.string().optional(), + }) + .optional(), + }) + .optional(), + style: z.object({ + all: z.array(z.string()), + chat: z.array(z.string()), + post: z.array(z.string()), + }), +}); + +// Encryption utility instance +const encryptionUtil = new EncryptionUtil( + process.env.ENCRYPTION_KEY || "default-key" +); + +/** + * Initialise SQLite database + */ +function initDatabase() { + db.exec(` + CREATE TABLE IF NOT EXISTS characters ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + characterState TEXT NOT NULL, + secretsIV TEXT NOT NULL, + createdAt TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updatedAt TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + `); + + console.log("Database initialised at:", dbDir); +} + +/** + * Inserts or updates a character in the SQLite database + * @param {Object} character - Validated character object + */ +function insertCharacter(character) { + try { + // Generate UUID from name + const id = encryptionUtil.stringToUuid(character.name); + + // Extract and encrypt secrets if they exist + let secretsIV = {}; + let characterState = { ...character }; + + if (character.settings?.secrets) { + const secretEntries = Object.entries(character.settings.secrets); + characterState.settings = { + ...character.settings, + secrets: {}, + }; + + for (const [key, value] of secretEntries) { + const encrypted = encryptionUtil.encrypt(value); + characterState.settings.secrets[key] = encrypted.encryptedText; + secretsIV[key] = encrypted.iv; + } + } + + // Upsert query using better-sqlite3 + const query = ` + INSERT INTO characters (id, name, characterState, secretsIV, createdAt, updatedAt) + VALUES (?, ?, ?, ?, datetime('now'), datetime('now')) + ON CONFLICT(id) DO UPDATE SET + characterState = excluded.characterState, + secretsIV = excluded.secretsIV, + updatedAt = datetime('now') + `; + + db.prepare(query).run( + id, + character.name, + JSON.stringify(characterState), + JSON.stringify(secretsIV) + ); + + console.log(`Successfully imported character: ${character.name}`); + } catch (error) { + console.error(`Error importing character ${character.name}:`, error); + throw error; + } +} + +/** + * Processes a single JSON file + * @param {string} filePath - Path to JSON file + */ +async function processJsonFile(filePath) { + try { + const content = await fs.readFile(filePath, "utf8"); + const character = JSON.parse(content); + + // Validate against schema + const validatedCharacter = CharacterSchema.parse(character); + + insertCharacter(validatedCharacter); + } catch (error) { + console.error(`Error processing file ${filePath}:`, error); + } +} + +/** + * Processes input path (file or directory) + * @param {string} inputPath - Path to process + */ +async function processPath(inputPath) { + try { + const stats = await fs.stat(inputPath); + + if (stats.isDirectory()) { + const files = await fs.readdir(inputPath); + for (const file of files) { + if (file.endsWith(".json")) { + await processJsonFile(path.join(inputPath, file)); + } + } + } else if (stats.isFile() && inputPath.endsWith(".json")) { + await processJsonFile(inputPath); + } + } catch (error) { + console.error("Error processing path:", error); + } +} + +// Main Usage +const inputPath = process.env.INPUT_PATH; +if (!inputPath) { + console.error("Please provide a path to a JSON file or directory"); + process.exit(1); +} + +initDatabase(); + +processPath(inputPath).catch((error) => { + console.error("Failed to process:", error); + process.exit(1); +}); From 394ccac307850cdafdaa2b085e0bd4ea5e95b232 Mon Sep 17 00:00:00 2001 From: Varkrishin Date: Sat, 23 Nov 2024 15:47:57 +0530 Subject: [PATCH 2/2] feat: loading characters from db at load and runtime --- .env.example | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.env.example b/.env.example index c2d1e182193..f74319868a2 100644 --- a/.env.example +++ b/.env.example @@ -91,4 +91,10 @@ STARKNET_ADDRESS= STARKNET_PRIVATE_KEY= STARKNET_RPC_URL= +# runtime management of character agents +FETCH_FROM_DB=false #During startup, fetch the characters from the database +ENCRYPTION_KEY= #mandatory field if FETCH_FROM_DB or AGENT_RUNTIME_MANAGEMENT is true, + #used to encrypt the secrets of characters +AGENT_RUNTIME_MANAGEMENT=false #Enable runtime management of character agents +AGENT_PORT=3001 #port for the runtime management of character agents if empty default 3001