feat: loading characters from db at load and runtime

Author: VarKrishin

agent/src/index.ts

@@ -33,6 +33,13 @@ import path from "path";
 import { fileURLToPath } from "url";
 import { character } from "./character.ts";
 import type { DirectClient } from "@ai16z/client-direct";
+import { Pool } from "pg";
+import { EncryptionUtil } from "@ai16z/eliza";
+import express, { Request as ExpressRequest } from "express";
+import bodyParser from "body-parser";
+import cors from "cors";
+
+let globalDirectClient: DirectClient | null = null;
 
 const __filename = fileURLToPath(import.meta.url); // get the resolved path to the file
 const __dirname = path.dirname(__filename); // get the name of the directory
@@ -306,8 +313,17 @@ async function startAgent(character: Character, directClient: DirectClient) {
     }
 }
 
+export function setGlobalDirectClient(client: DirectClient) {
+    globalDirectClient = client;
+}
+
+export function getGlobalDirectClient(): DirectClient | null {
+    return globalDirectClient;
+}
+
 const startAgents = async () => {
     const directClient = await DirectClientInterface.start();
+    setGlobalDirectClient(directClient as DirectClient);
     const args = parseArguments();
 
     let charactersArg = args.characters || args.character;
@@ -318,6 +334,15 @@ const startAgents = async () => {
         characters = await loadCharacters(charactersArg);
     }
 
+    const shouldFetchFromDb = process.env.FETCH_FROM_DB === "true";
+
+    if (shouldFetchFromDb) {
+        characters = await loadCharactersFromDb(charactersArg);
+        if (characters.length === 0) {
+            characters = [character];
+        }
+    }
+
     try {
         for (const character of characters) {
             await startAgent(character, directClient as DirectClient);
@@ -384,3 +409,174 @@ async function handleUserInput(input, agentId) {
         console.error("Error fetching response:", error);
     }
 }
+
+/**
+ * Loads characters from PostgreSQL database
+ * @param characterNames - Optional comma-separated list of character names to load
+ * @returns Promise of loaded and decrypted characters
+ */
+export async function loadCharactersFromDb(
+    characterNames?: string
+): Promise<Character[]> {
+    try {
+        const encryptionUtil = new EncryptionUtil(
+            process.env.ENCRYPTION_KEY || "default-key"
+        );
+
+        const dataDir = path.join(__dirname, "../data");
+
+        if (!fs.existsSync(dataDir)) {
+            fs.mkdirSync(dataDir, { recursive: true });
+        }
+
+        const db = initializeDatabase(dataDir);
+        await db.init();
+
+        // Convert names to UUIDs if provided
+        const characterIds = characterNames
+            ?.split(",")
+            .map((name) => name.trim())
+            .map((name) => stringToUuid(name));
+
+        // Get characters and their secretsIVs from database
+        const [characters, secretsIVs] = await db.loadCharacters(characterIds);
+
+        if (characters.length === 0) {
+            elizaLogger.log(
+                "No characters found in database, using default character"
+            );
+            return [];
+        }
+
+        // Process each character with its corresponding secretsIV
+        const processedCharacters = await Promise.all(
+            characters.map(async (character, index) => {
+                try {
+                    // Decrypt secrets if they exist
+                    if (character.settings?.secrets) {
+                        const decryptedSecrets: { [key: string]: string } = {};
+                        const secretsIV = secretsIVs[index];
+
+                        for (const [key, encryptedValue] of Object.entries(
+                            character.settings.secrets
+                        )) {
+                            const iv = secretsIV[key];
+                            if (!iv) {
+                                elizaLogger.error(
+                                    `Missing IV for secret ${key} in character ${character.name}`
+                                );
+                                continue;
+                            }
+
+                            try {
+                                decryptedSecrets[key] = encryptionUtil.decrypt({
+                                    encryptedText: encryptedValue,
+                                    iv,
+                                });
+                            } catch (error) {
+                                elizaLogger.error(
+                                    `Failed to decrypt secret ${key} for character ${character.name}:`,
+                                    error
+                                );
+                            }
+                        }
+                        character.settings.secrets = decryptedSecrets;
+                    }
+
+                    // Handle plugins
+                    if (character.plugins) {
+                        elizaLogger.log("Plugins are: ", character.plugins);
+                        const importedPlugins = await Promise.all(
+                            character.plugins.map(async (plugin) => {
+                                // if the plugin name doesnt start with @eliza,
+
+                                const importedPlugin = await import(
+                                    plugin.name
+                                );
+                                return importedPlugin;
+                            })
+                        );
+
+                        character.plugins = importedPlugins;
+                    }
+
+                    validateCharacterConfig(character);
+                    elizaLogger.log(
+                        `Character loaded from db: ${character.name}`
+                    );
+                    console.log("-------------------------------");
+                    return character;
+                } catch (error) {
+                    elizaLogger.error(
+                        `Error processing character ${character.name}:`,
+                        error
+                    );
+                    throw error;
+                }
+            })
+        );
+
+        return processedCharacters;
+    } catch (error) {
+        elizaLogger.error("Database error:", error);
+        elizaLogger.log("Falling back to default character");
+        return [defaultCharacter];
+    }
+}
+
+// If dynamic loading is enabled, start the express server
+// we can directly call this endpoint to load an agent
+// otherwise we can use the direct client as a proxy if we
+// want to expose only single post to public
+if (process.env.AGENT_RUNTIME_MANAGEMENT === "true") {
+    const app = express();
+    app.use(cors());
+    app.use(bodyParser.json());
+
+    // This endpoint can be directly called or
+    app.post(
+        "/load/:agentName",
+        async (req: ExpressRequest, res: express.Response) => {
+            try {
+                const agentName = req.params.agentName;
+                const characters = await loadCharactersFromDb(agentName);
+
+                if (characters.length === 0) {
+                    res.status(404).json({
+                        success: false,
+                        error: `Character ${agentName} does not exist in DB`,
+                    });
+                    return;
+                }
+
+                const directClient = getGlobalDirectClient();
+                await startAgent(characters[0], directClient);
+
+                res.json({
+                    success: true,
+                    port: settings.SERVER_PORT,
+                    character: {
+                        id: characters[0].id,
+                        name: characters[0].name,
+                    },
+                });
+            } catch (error) {
+                elizaLogger.error(`Error loading agent:`, error);
+                res.status(500).json({
+                    success: false,
+                    error: error.message,
+                });
+            }
+        }
+    );
+
+    const agentPort = settings.AGENT_PORT
+        ? parseInt(settings.AGENT_PORT)
+        : 3001;
+    //if agent port is 0, it means we want to use a random port
+    const server = app.listen(agentPort, () => {
+        elizaLogger.success(
+            `Agent server running at http://localhost:${agentPort}/`
+        );
+    });
+}

docs/docs/guides/template-configuration.md

@@ -15,14 +15,14 @@ Here are all the template options you can configure:
 ```json
 {
   "templates": {
-    "goalsTemplate": "",               // Define character goals
-    "factsTemplate": "",              // Specify character knowledge
-    "messageHandlerTemplate": "",      // Handle general messages
-    "shouldRespondTemplate": "",       // Control response triggers
+    "goalsTemplate": "", // Define character goals
+    "factsTemplate": "", // Specify character knowledge
+    "messageHandlerTemplate": "", // Handle general messages
+    "shouldRespondTemplate": "", // Control response triggers
     "continueMessageHandlerTemplate": "", // Manage conversation flow
-    "evaluationTemplate": "",         // Handle response evaluation
-    "twitterSearchTemplate": "",      // Process Twitter searches
-    "twitterPostTemplate": "",        // Format Twitter posts
+    "evaluationTemplate": "", // Handle response evaluation
+    "twitterSearchTemplate": "", // Process Twitter searches
+    "twitterPostTemplate": "", // Format Twitter posts
     "twitterMessageHandlerTemplate": "", // Handle Twitter messages
     "twitterShouldRespondTemplate": "", // Control Twitter responses
     "telegramMessageHandlerTemplate": "", // Handle Telegram messages
@@ -60,11 +60,11 @@ Configure platform-specific behaviors for your character, such as handling direc
   "clientConfig": {
     "telegram": {
       "shouldIgnoreDirectMessages": true, // Ignore DMs
-      "shouldIgnoreBotMessages": true    // Ignore bot messages
+      "shouldIgnoreBotMessages": true // Ignore bot messages
     },
     "discord": {
-      "shouldIgnoreBotMessages": true,   // Ignore bot messages
-      "shouldIgnoreDirectMessages": true  // Ignore DMs
+      "shouldIgnoreBotMessages": true, // Ignore bot messages
+      "shouldIgnoreDirectMessages": true // Ignore DMs
     }
   }
 }
@@ -73,11 +73,13 @@ Configure platform-specific behaviors for your character, such as handling direc
 ## Best Practices
 
 1. **Template Management**
+
    - Keep templates focused and specific
    - Use clear, consistent formatting
    - Document custom template behavior
 
 2. **Client Configuration**
+
    - Configure per platform as needed
    - Test behavior in development
    - Monitor interaction patterns

packages/adapter-postgres/schema.sql

@@ -26,6 +26,15 @@ CREATE TABLE IF NOT EXISTS accounts (
     "details" JSONB DEFAULT '{}'::jsonb
 );
 
+CREATE TABLE IF NOT EXISTS characters (
+    "id" UUID PRIMARY KEY,
+    "name" TEXT,
+    "characterState" JSONB NOT NULL,
+    "secretsIV" JSONB DEFAULT '{}'::jsonb,
+    "createdAt" TIMESTAMP DEFAULT now(),
+    "updatedAt" TIMESTAMP DEFAULT now()
+);
+
 CREATE TABLE IF NOT EXISTS rooms (
     "id" UUID PRIMARY KEY,
     "createdAt" TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP

packages/adapter-postgres/src/index.ts

@@ -9,6 +9,10 @@ import {
     type Relationship,
     type UUID,
     type IDatabaseCacheAdapter,
+    type IDatabaseAdapter,
+    type CharacterTable,
+    type Secrets,
+    type Character,
     Participant,
     DatabaseAdapter,
     elizaLogger,
@@ -22,7 +26,7 @@ const __dirname = path.dirname(__filename); // get the name of the directory
 
 export class PostgresDatabaseAdapter
     extends DatabaseAdapter<Pool>
-    implements IDatabaseCacheAdapter
+    implements IDatabaseCacheAdapter, IDatabaseAdapter
 {
     private pool: Pool;
 
@@ -39,6 +43,7 @@ export class PostgresDatabaseAdapter
             ...defaultConfig,
             ...connectionConfig, // Allow overriding defaults
         });
+        this.db = this.pool;
 
         this.pool.on("error", async (err) => {
             elizaLogger.error("Unexpected error on idle client", err);
@@ -736,7 +741,9 @@ export class PostgresDatabaseAdapter
             );
 
             if (existingParticipant.rows.length > 0) {
-                console.log(`Participant with userId ${userId} already exists in room ${roomId}.`);
+                console.log(
+                    `Participant with userId ${userId} already exists in room ${roomId}.`
+                );
                 return; // Exit early if the participant already exists
             }
 
@@ -750,11 +757,13 @@ export class PostgresDatabaseAdapter
         } catch (error) {
             // This is to prevent duplicate participant error in case of a race condition
             // Handle unique constraint violation error (code 23505)
-            if (error.code === '23505') {
-                console.warn(`Participant with userId ${userId} already exists in room ${roomId}.`);               // Optionally, you can log this or handle it differently
+            if (error.code === "23505") {
+                console.warn(
+                    `Participant with userId ${userId} already exists in room ${roomId}.`
+                ); // Optionally, you can log this or handle it differently
             } else {
                 // Handle other errors
-                console.error('Error adding participant:', error);
+                console.error("Error adding participant:", error);
                 return false;
             }
         } finally {
@@ -958,6 +967,48 @@ export class PostgresDatabaseAdapter
             client.release();
         }
     }
+
+    /**
+     * Loads characters from database
+     * @param characterIds Optional array of character UUIDs to load
+     * @returns Promise of tuple containing Characters array and their corresponding SecretsIV
+     */
+    async loadCharacters(
+        characterIds?: UUID[]
+    ): Promise<[Character[], Secrets[]]> {
+        const client = await this.pool.connect();
+        try {
+            let query =
+                'SELECT "id", "name", "characterState", "secretsIV" FROM characters';
+            const queryParams: any[] = [];
+
+            if (characterIds?.length) {
+                query += ' WHERE "id" = ANY($1)';
+                queryParams.push(characterIds);
+            }
+
+            query += " ORDER BY name";
+            const result = await client.query<CharacterTable>(
+                query,
+                queryParams
+            );
+
+            const characters: Character[] = [];
+            const secretsIVs: Secrets[] = [];
+
+            for (const row of result.rows) {
+                characters.push(row.characterState);
+                secretsIVs.push(row.secretsIV || {});
+            }
+
+            return [characters, secretsIVs];
+        } catch (error) {
+            elizaLogger.error("Error loading characters:", error);
+            throw error;
+        } finally {
+            client.release();
+        }
+    }
 }
 
 export default PostgresDatabaseAdapter;