feat: custom s3 endpoint url for 'plugin-node'

elizaOS · Jan 13, 2025 · 0f0e3f4 · 0f0e3f4
1 parent 4420a7a
commit 0f0e3f4
Show file tree

Hide file tree

Showing 5 changed files with 4,009 additions and 11,770 deletions.
diff --git a/.env.example b/.env.example
@@ -523,6 +523,7 @@ AWS_SECRET_ACCESS_KEY=
 AWS_REGION=
 AWS_S3_BUCKET=
 AWS_S3_UPLOAD_PATH=
+AWS_S3_ENDPOINT= # Optional custom S3 endpoint
 
 # Deepgram
 DEEPGRAM_API_KEY=

diff --git a/packages/plugin-node/README.md b/packages/plugin-node/README.md
@@ -54,6 +54,7 @@ AWS_SECRET_ACCESS_KEY=your_aws_secret_key
 AWS_REGION=your_aws_region
 AWS_S3_BUCKET=your_s3_bucket
 AWS_S3_UPLOAD_PATH=your_upload_path
+AWS_S3_ENDPOINT=your_custom_s3_endpoint # Optional (e.g., if using local S3)
 ```
 
 ## Usage
@@ -72,7 +73,7 @@ elizaos.registerPlugin(nodePlugin);
 
 ### AwsS3Service
 
-Handles file uploads and management with AWS S3.
+Handles file uploads and management with AWS S3 or S3-compatible tools.
 
 ### BrowserService
 
@@ -358,7 +359,7 @@ This plugin integrates with and builds upon several key technologies:
 - [FFmpeg](https://ffmpeg.org/) - Media processing capabilities
 - [ElevenLabs](https://elevenlabs.io/) - Voice synthesis
 - [OpenAI](https://openai.com/) - Transcription and AI services
-- [AWS S3](https://aws.amazon.com/s3/) - Cloud storage
+- [AWS S3](https://aws.amazon.com/s3/) - Cloud storage (or other S3-compatible tools)
 - [Playwright](https://playwright.dev/) - Browser automation
 - [LLaMA](https://github.com/facebookresearch/llama) - Local language models
 - [VITS](https://github.com/jaywalnut310/vits) - Voice synthesis

diff --git a/packages/plugin-node/src/environment.ts b/packages/plugin-node/src/environment.ts
@@ -81,6 +81,7 @@ export async function validateNodeConfig(
                 AWS_REGION: runtime.getSetting("AWS_REGION"),
                 AWS_S3_BUCKET: runtime.getSetting("AWS_S3_BUCKET"),
                 AWS_S3_UPLOAD_PATH: runtime.getSetting("AWS_S3_UPLOAD_PATH"),
+                AWS_S3_ENDPOINT: runtime.getSetting("AWS_S3_ENDPOINT"),
             }),
         };
 

diff --git a/packages/plugin-node/src/services/awsS3.ts b/packages/plugin-node/src/services/awsS3.ts
@@ -9,6 +9,7 @@ import {
     GetObjectCommand,
     PutObjectCommand,
     S3Client,
+    S3ClientConfig,
 } from "@aws-sdk/client-s3";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 import * as fs from "fs";
@@ -48,6 +49,7 @@ export class AwsS3Service extends Service implements IAwsS3Service {
         );
         const AWS_REGION = this.runtime.getSetting("AWS_REGION");
         const AWS_S3_BUCKET = this.runtime.getSetting("AWS_S3_BUCKET");
+        const AWS_S3_ENDPOINT = this.runtime.getSetting("AWS_S3_ENDPOINT");
 
         if (
             !AWS_ACCESS_KEY_ID ||
@@ -58,13 +60,19 @@ export class AwsS3Service extends Service implements IAwsS3Service {
             return false;
         }
 
-        this.s3Client = new S3Client({
+        const clientConfig: S3ClientConfig = {
             region: AWS_REGION,
             credentials: {
                 accessKeyId: AWS_ACCESS_KEY_ID,
                 secretAccessKey: AWS_SECRET_ACCESS_KEY,
             },
-        });
+        };
+        if (AWS_S3_ENDPOINT) {
+            clientConfig.endpoint = AWS_S3_ENDPOINT;
+            clientConfig.forcePathStyle = true;
+        }
+
+        this.s3Client = new S3Client(clientConfig);
         this.bucket = AWS_S3_BUCKET;
         return true;
     }
@@ -117,7 +125,14 @@ export class AwsS3Service extends Service implements IAwsS3Service {
 
             // If not using signed URL, return public access URL
             if (!useSignedUrl) {
-                result.url = `https://${this.bucket}.s3.${process.env.AWS_REGION}.amazonaws.com/${fileName}`;
+                // Use the endpoint directly (e.g., local S3) if a custom value was set
+                if (this.s3Client.config.endpoint) {
+                    const endpoint = await this.s3Client.config.endpoint();
+                    const port = endpoint.port ? `:${endpoint.port}` : "";
+                    result.url = `${endpoint.protocol}//${endpoint.hostname}${port}${endpoint.path}${this.bucket}/${fileName}`;
+                } else {
+                    result.url = `https://${this.bucket}.s3.${process.env.AWS_REGION}.amazonaws.com/${fileName}`;
+                }
             } else {
                 const getObjectCommand = new GetObjectCommand({
                     Bucket: this.bucket,
@@ -239,7 +254,14 @@ export class AwsS3Service extends Service implements IAwsS3Service {
 
             // Return corresponding URL based on requirements
             if (!useSignedUrl) {
-                result.url = `https://${this.bucket}.s3.${process.env.AWS_REGION}.amazonaws.com/${key}`;
+                // Use the endpoint directly (e.g., local S3) if a custom value was set
+                if (this.s3Client.config.endpoint) {
+                    const endpoint = await this.s3Client.config.endpoint();
+                    const port = endpoint.port ? `:${endpoint.port}` : "";
+                    result.url = `${endpoint.protocol}//${endpoint.hostname}${port}${endpoint.path}${this.bucket}/${key}`;
+                } else {
+                    result.url = `https://${this.bucket}.s3.${process.env.AWS_REGION}.amazonaws.com/${key}`;
+                }
             } else {
                 const getObjectCommand = new GetObjectCommand({
                     Bucket: this.bucket,