AWS secrets manager integration in settings

Author: ropresearch

packages/core/package.json

@@ -56,6 +56,8 @@
     "@ai-sdk/groq": "0.0.3",
     "@ai-sdk/openai": "1.0.5",
     "@anthropic-ai/sdk": "0.30.1",
+    "@aws-sdk/client-secrets-manager": "*",
+    "@aws-sdk/client-sts": "*",
     "@fal-ai/client": "1.2.0",
     "@types/uuid": "10.0.0",
     "ai": "3.4.33",

packages/core/src/settings.ts

@@ -1,22 +1,18 @@
 import { config } from "dotenv";
 import fs from "fs";
 import path from "path";
+import * as dotenv from 'dotenv';
 import elizaLogger from "./logger.ts";
-
-elizaLogger.info("Loading embedding settings:", {
-    USE_OPENAI_EMBEDDING: process.env.USE_OPENAI_EMBEDDING,
-    USE_OLLAMA_EMBEDDING: process.env.USE_OLLAMA_EMBEDDING,
-    OLLAMA_EMBEDDING_MODEL:
-        process.env.OLLAMA_EMBEDDING_MODEL || "mxbai-embed-large",
-});
-
-// Add this logging block
-elizaLogger.info("Loading character settings:", {
-    CHARACTER_PATH: process.env.CHARACTER_PATH,
-    ARGV: process.argv,
-    CHARACTER_ARG: process.argv.find((arg) => arg.startsWith("--character=")),
-    CWD: process.cwd(),
-});
+import {
+    SecretsManagerClient,
+    GetSecretValueCommand
+} from "@aws-sdk/client-secrets-manager";
+
+interface SecretConfig {
+    secretNames: string[];
+    region: string;
+    roleArn?: string;
+}
 
 interface Settings {
     [key: string]: string | undefined;
@@ -26,84 +22,163 @@ let environmentSettings: Settings = {};
 
 /**
  * Determines if code is running in a browser environment
- * @returns {boolean} True if in browser environment
  */
 const isBrowser = (): boolean => {
-    return (
-        typeof window !== "undefined" && typeof window.document !== "undefined"
-    );
+    return typeof window !== "undefined" && typeof window.document !== "undefined";
 };
 
 /**
  * Recursively searches for a .env file starting from the current directory
- * and moving up through parent directories (Node.js only)
- * @param {string} [startDir=process.cwd()] - Starting directory for the search
- * @returns {string|null} Path to the nearest .env file or null if not found
  */
 export function findNearestEnvFile(startDir = process.cwd()) {
     if (isBrowser()) return null;
 
+    elizaLogger.info('Starting search for .env file from directory:', startDir);
     let currentDir = startDir;
 
-    // Continue searching until we reach the root directory
     while (currentDir !== path.parse(currentDir).root) {
         const envPath = path.join(currentDir, ".env");
+        elizaLogger.info('Checking for .env file at:', envPath);
 
         if (fs.existsSync(envPath)) {
+            elizaLogger.info('Found .env file at:', envPath);
             return envPath;
         }
 
-        // Move up to parent directory
         currentDir = path.dirname(currentDir);
+        elizaLogger.info('Moving up to directory:', currentDir);
     }
 
-    // Check root directory as well
     const rootEnvPath = path.join(path.parse(currentDir).root, ".env");
+    elizaLogger.info('Checking root directory for .env file:', rootEnvPath);
     return fs.existsSync(rootEnvPath) ? rootEnvPath : null;
 }
 
+/**
+ * Fetches secrets from AWS Secrets Manager
+ */
+async function getSecretsManagerValues(secretConfig: SecretConfig): Promise<Record<string, string>> {
+    elizaLogger.info('Starting to fetch secrets with config:', secretConfig);
+    const { secretNames, region } = secretConfig;
+
+    const client = new SecretsManagerClient({ region });
+    const secrets: Record<string, string> = {};
+
+    elizaLogger.info('Starting to fetch secrets for:', secretNames);
+    const secretPromises = secretNames.map(async (secretName) => {
+        elizaLogger.info('Fetching secret:', secretName);
+        try {
+            const response = await client.send(
+                new GetSecretValueCommand({ SecretId: secretName })
+            );
+            elizaLogger.info(`Received response for secret ${secretName}`);
+
+            if (response.SecretString) {
+                try {
+                    elizaLogger.info(`Attempting to parse secret ${secretName} as JSON`);
+                    const secretJson = JSON.parse(response.SecretString);
+                    elizaLogger.info(`Successfully parsed secret ${secretName}, found keys:`, Object.keys(secretJson));
+                    Object.entries(secretJson).forEach(([key, value]) => {
+                        secrets[key] = value as string;
+                    });
+                } catch (parseError) {
+                    elizaLogger.info(`Secret ${secretName} is not JSON, storing as plain string`);
+                    secrets[secretName] = response.SecretString;
+                }
+            }
+        } catch (error) {
+            elizaLogger.error(`Failed to fetch secret ${secretName}:`, error);
+            throw error;
+        }
+    });
+
+    await Promise.all(secretPromises);
+    return secrets;
+}
+
 /**
  * Configures environment settings for browser usage
- * @param {Settings} settings - Object containing environment variables
  */
 export function configureSettings(settings: Settings) {
     environmentSettings = { ...settings };
 }
 
 /**
- * Loads environment variables from the nearest .env file in Node.js
- * or returns configured settings in browser
- * @returns {Settings} Environment variables object
- * @throws {Error} If no .env file is found in Node.js environment
+ * Loads environment variables from .env and AWS Secrets Manager
  */
-export function loadEnvConfig(): Settings {
+export async function loadEnvConfig(): Promise<Settings> {
     // For browser environments, return the configured settings
     if (isBrowser()) {
         return environmentSettings;
     }
 
-    // Node.js environment: load from .env file
+    elizaLogger.info('Starting loadEnvConfig');
     const envPath = findNearestEnvFile();
 
-    // attempt to Load the .env file into process.env
-    const result = config(envPath ? { path: envPath } : {});
+    if (!envPath) {
+        elizaLogger.error('No .env file found in directory hierarchy');
+        throw new Error("No .env file found in current or parent directories.");
+    }
 
-    if (!result.error) {
-        console.log(`Loaded .env file from: ${envPath}`);
+    elizaLogger.info('Loading .env file from:', envPath);
+    const result = config({ path: envPath });
+
+    if (result.error) {
+        elizaLogger.error('Error loading .env file:', result.error);
+        throw new Error(`Error loading .env file: ${result.error}`);
     }
+
+    // Read directly from the .env file
+    const envConfig = dotenv.parse(fs.readFileSync(envPath));
+    elizaLogger.info('Direct .env file contents:', envConfig);
+
+    // Create secretConfig using the direct .env contents
+    const secretConfig = {
+        secretNames: envConfig.SECRET_NAMES?.split(',') || [],
+        region: envConfig.AWS_REGION || 'us-east-1',
+        roleArn: envConfig.AWS_ROLE_ARN?.trim()
+    };
+
+    // Load secrets from AWS if configured
+    if (secretConfig.secretNames.length > 0) {
+        try {
+            const secrets = await getSecretsManagerValues(secretConfig);
+            elizaLogger.info('Successfully fetched secrets, found keys:', Object.keys(secrets));
+
+            Object.entries(secrets).forEach(([key, value]) => {
+                process.env[key] = value;
+            });
+            elizaLogger.info(`Loaded all secrets from AWS Secrets Manager: ${secretConfig.secretNames.join(', ')}`);
+        } catch (error) {
+            elizaLogger.error('Failed to load secrets from AWS Secrets Manager:', error);
+            throw new Error(`Failed to load secrets from AWS Secrets Manager: ${error}`);
+        }
+    } else {
+        elizaLogger.info('No secret names provided, skipping AWS Secrets Manager');
+    }
+
+    // Log embedding settings
+    elizaLogger.info("Loading embedding settings:", {
+        USE_OPENAI_EMBEDDING: process.env.USE_OPENAI_EMBEDDING,
+        USE_OLLAMA_EMBEDDING: process.env.USE_OLLAMA_EMBEDDING,
+        OLLAMA_EMBEDDING_MODEL: process.env.OLLAMA_EMBEDDING_MODEL || "mxbai-embed-large",
+    });
+
+    // Log character settings
+    elizaLogger.info("Loading character settings:", {
+        CHARACTER_PATH: process.env.CHARACTER_PATH,
+        ARGV: process.argv,
+        CHARACTER_ARG: process.argv.find((arg) => arg.startsWith("--character=")),
+        CWD: process.cwd(),
+    });
+
     return process.env as Settings;
 }
 
 /**
  * Gets a specific environment variable
- * @param {string} key - The environment variable key
- * @param {string} [defaultValue] - Optional default value if key doesn't exist
- * @returns {string|undefined} The environment variable value or default value
  */
-export function getEnvVariable(
-    key: string,
-    defaultValue?: string
-): string | undefined {
+export function getEnvVariable(key: string, defaultValue?: string): string | undefined {
     if (isBrowser()) {
         return environmentSettings[key] || defaultValue;
     }
@@ -112,8 +187,6 @@ export function getEnvVariable(
 
 /**
  * Checks if a specific environment variable exists
- * @param {string} key - The environment variable key
- * @returns {boolean} True if the environment variable exists
  */
 export function hasEnvVariable(key: string): boolean {
     if (isBrowser()) {
@@ -123,15 +196,14 @@ export function hasEnvVariable(key: string): boolean {
 }
 
 // Initialize settings based on environment
-export const settings = isBrowser() ? environmentSettings : loadEnvConfig();
+export const settings = isBrowser() ? environmentSettings : await loadEnvConfig();
 
 elizaLogger.info("Parsed settings:", {
     USE_OPENAI_EMBEDDING: settings.USE_OPENAI_EMBEDDING,
     USE_OPENAI_EMBEDDING_TYPE: typeof settings.USE_OPENAI_EMBEDDING,
     USE_OLLAMA_EMBEDDING: settings.USE_OLLAMA_EMBEDDING,
     USE_OLLAMA_EMBEDDING_TYPE: typeof settings.USE_OLLAMA_EMBEDDING,
-    OLLAMA_EMBEDDING_MODEL:
-        settings.OLLAMA_EMBEDDING_MODEL || "mxbai-embed-large",
+    OLLAMA_EMBEDDING_MODEL: settings.OLLAMA_EMBEDDING_MODEL || "mxbai-embed-large",
 });
 
-export default settings;
+export default settings;