Merge pull request #1481 from 0xPBIT/images-in-chat-client

feat: add image features to react chat client

Author: shakkernerd

client/src/Chat.tsx

@@ -1,55 +1,75 @@
-import { useState } from "react";
+import { useRef, useState } from "react";
 import { useParams } from "react-router-dom";
 import { useMutation } from "@tanstack/react-query";
 import { Button } from "@/components/ui/button";
+import { ImageIcon } from "lucide-react";
 import { Input } from "@/components/ui/input";
 import "./App.css";
+import path from "path";
 
 type TextResponse = {
     text: string;
     user: string;
+    attachments?: { url: string; contentType: string; title: string }[];
 };
 
 export default function Chat() {
     const { agentId } = useParams();
     const [input, setInput] = useState("");
     const [messages, setMessages] = useState<TextResponse[]>([]);
+    const [selectedFile, setSelectedFile] = useState<File | null>(null);
+    const fileInputRef = useRef<HTMLInputElement>(null);
 
     const mutation = useMutation({
         mutationFn: async (text: string) => {
+            const formData = new FormData();
+            formData.append("text", text);
+            formData.append("userId", "user");
+            formData.append("roomId", `default-room-${agentId}`);
+
+            if (selectedFile) {
+                formData.append("file", selectedFile);
+            }
+
             const res = await fetch(`/api/${agentId}/message`, {
                 method: "POST",
-                headers: {
-                    "Content-Type": "application/json",
-                },
-                body: JSON.stringify({
-                    text,
-                    userId: "user",
-                    roomId: `default-room-${agentId}`,
-                }),
+                body: formData,
             });
             return res.json() as Promise<TextResponse[]>;
         },
         onSuccess: (data) => {
             setMessages((prev) => [...prev, ...data]);
+            setSelectedFile(null);
         },
     });
 
     const handleSubmit = async (e: React.FormEvent) => {
         e.preventDefault();
-        if (!input.trim()) return;
+        if (!input.trim() && !selectedFile) return;
 
         // Add user message immediately to state
         const userMessage: TextResponse = {
             text: input,
             user: "user",
+            attachments: selectedFile ? [{ url: URL.createObjectURL(selectedFile), contentType: selectedFile.type, title: selectedFile.name }] : undefined,
         };
         setMessages((prev) => [...prev, userMessage]);
 
         mutation.mutate(input);
         setInput("");
     };
 
+    const handleFileSelect = () => {
+        fileInputRef.current?.click();
+    };
+
+    const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+        const file = e.target.files?.[0];
+        if (file && file.type.startsWith('image/')) {
+            setSelectedFile(file);
+        }
+    };
+
     return (
         <div className="flex flex-col h-screen max-h-screen w-full">
             <div className="flex-1 min-h-0 overflow-y-auto p-4">
@@ -71,10 +91,23 @@ export default function Chat() {
                                             : "bg-muted"
                                     }`}
                                 >
-                                    <pre className="whitespace-pre-wrap break-words font-sans m-0">
-                                        {message.text}
-                                    </pre>
-                                </div>
+                                    {message.text}
+                                    {message.attachments?.map((attachment, i) => (
+                                        attachment.contentType.startsWith('image/') && (
+                                            <img
+                                                key={i}
+                                                src={message.user === "user"
+                                                    ? attachment.url
+                                                    : attachment.url.startsWith('http')
+                                                        ? attachment.url
+                                                        : `http://localhost:3000/media/generated/${attachment.url.split('/').pop()}`
+                                                }
+                                                alt={attachment.title || "Attached image"}
+                                                className="mt-2 max-w-full rounded-lg"
+                                            />
+                                        )
+                                    ))}
+                                 </div>
                             </div>
                         ))
                     ) : (
@@ -88,17 +121,38 @@ export default function Chat() {
             <div className="border-t p-4 bg-background">
                 <div className="max-w-3xl mx-auto">
                     <form onSubmit={handleSubmit} className="flex gap-2">
+                        <input
+                            type="file"
+                            ref={fileInputRef}
+                            onChange={handleFileChange}
+                            accept="image/*"
+                            className="hidden"
+                        />
                         <Input
                             value={input}
                             onChange={(e) => setInput(e.target.value)}
                             placeholder="Type a message..."
                             className="flex-1"
                             disabled={mutation.isPending}
                         />
+                        <Button
+                            type="button"
+                            variant="outline"
+                            size="icon"
+                            onClick={handleFileSelect}
+                            disabled={mutation.isPending}
+                        >
+                            <ImageIcon className="h-4 w-4" />
+                        </Button>
                         <Button type="submit" disabled={mutation.isPending}>
                             {mutation.isPending ? "..." : "Send"}
                         </Button>
                     </form>
+                    {selectedFile && (
+                        <div className="mt-2 text-sm text-muted-foreground">
+                            Selected file: {selectedFile.name}
+                        </div>
+                    )}
                 </div>
             </div>
         </div>

packages/client-direct/package.json

@@ -17,7 +17,8 @@
         "multer": "1.4.5-lts.1"
     },
     "devDependencies": {
-        "tsup": "8.3.5"
+        "tsup": "8.3.5",
+        "@types/multer": "^1.4.12"
     },
     "scripts": {
         "build": "tsup --format esm --dts",

packages/client-direct/src/index.ts

@@ -1,12 +1,13 @@
 import bodyParser from "body-parser";
 import cors from "cors";
 import express, { Request as ExpressRequest } from "express";
-import multer, { File } from "multer";
+import multer from "multer";
 import {
     elizaLogger,
     generateCaption,
     generateImage,
-    getEmbeddingZeroVector,
+    Media,
+    getEmbeddingZeroVector
 } from "@elizaos/core";
 import { composeContext } from "@elizaos/core";
 import { generateMessageResponse } from "@elizaos/core";
@@ -24,7 +25,23 @@ import { settings } from "@elizaos/core";
 import { createApiRouter } from "./api.ts";
 import * as fs from "fs";
 import * as path from "path";
-const upload = multer({ storage: multer.memoryStorage() });
+
+const storage = multer.diskStorage({
+    destination: (req, file, cb) => {
+        const uploadDir = path.join(process.cwd(), "data", "uploads");
+        // Create the directory if it doesn't exist
+        if (!fs.existsSync(uploadDir)) {
+            fs.mkdirSync(uploadDir, { recursive: true });
+        }
+        cb(null, uploadDir);
+    },
+    filename: (req, file, cb) => {
+        const uniqueSuffix = `${Date.now()}-${Math.round(Math.random() * 1e9)}`;
+        cb(null, `${uniqueSuffix}-${file.originalname}`);
+    },
+});
+
+const upload = multer({ storage });
 
 export const messageHandlerTemplate =
     // {{goals}}
@@ -71,12 +88,22 @@ export class DirectClient {
         this.app.use(bodyParser.json());
         this.app.use(bodyParser.urlencoded({ extended: true }));
 
+        // Serve both uploads and generated images
+        this.app.use(
+            "/media/uploads",
+            express.static(path.join(process.cwd(), "/data/uploads"))
+        );
+        this.app.use(
+            "/media/generated",
+            express.static(path.join(process.cwd(), "/generatedImages"))
+        );
+
         const apiRouter = createApiRouter(this.agents, this);
         this.app.use(apiRouter);
 
         // Define an interface that extends the Express Request interface
         interface CustomRequest extends ExpressRequest {
-            file: File;
+            file?: Express.Multer.File;
         }
 
         // Update the route handler to use CustomRequest instead of express.Request
@@ -133,6 +160,7 @@ export class DirectClient {
 
         this.app.post(
             "/:agentId/message",
+            upload.single("file"),
             async (req: express.Request, res: express.Response) => {
                 const agentId = req.params.agentId;
                 const roomId = stringToUuid(
@@ -167,9 +195,29 @@ export class DirectClient {
                 const text = req.body.text;
                 const messageId = stringToUuid(Date.now().toString());
 
+                const attachments: Media[] = [];
+                if (req.file) {
+                    const filePath = path.join(
+                        process.cwd(),
+                        "agent",
+                        "data",
+                        "uploads",
+                        req.file.filename
+                    );
+                    attachments.push({
+                        id: Date.now().toString(),
+                        url: filePath,
+                        title: req.file.originalname,
+                        source: "direct",
+                        description: `Uploaded file: ${req.file.originalname}`,
+                        text: "",
+                        contentType: req.file.mimetype,
+                    });
+                }
+
                 const content: Content = {
                     text,
-                    attachments: [],
+                    attachments,
                     source: "direct",
                     inReplyTo: undefined,
                 };