Skip to content

Commit 4832d1f

Browse files
odilitimeodilitime
authoredDec 26, 2024
Merge pull request #1446 from tomguluson92/main
feat: Add Text to 3D function
2 parents ce9ccfc + f477dcf commit 4832d1f

File tree

9 files changed

+272
-0
lines changed

9 files changed

+272
-0
lines changed
 

‎agent/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
"@elizaos/plugin-multiversx": "workspace:*",
5353
"@elizaos/plugin-near": "workspace:*",
5454
"@elizaos/plugin-zksync-era": "workspace:*",
55+
"@elizaos/plugin-3d-generation": "workspace:*",
5556
"readline": "1.3.0",
5657
"ws": "8.18.0",
5758
"yargs": "17.7.2"

‎agent/src/index.ts

+4
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ import { evmPlugin } from "@elizaos/plugin-evm";
4646
import { storyPlugin } from "@elizaos/plugin-story";
4747
import { flowPlugin } from "@elizaos/plugin-flow";
4848
import { imageGenerationPlugin } from "@elizaos/plugin-image-generation";
49+
import { ThreeDGenerationPlugin } from "@elizaos/plugin-3d-generation";
4950
import { multiversxPlugin } from "@elizaos/plugin-multiversx";
5051
import { nearPlugin } from "@elizaos/plugin-near";
5152
import { nftGenerationPlugin } from "@elizaos/plugin-nft-generation";
@@ -516,6 +517,9 @@ export async function createAgent(
516517
getSecret(character, "HEURIST_API_KEY")
517518
? imageGenerationPlugin
518519
: null,
520+
getSecret(character, "FAL_API_KEY")
521+
? ThreeDGenerationPlugin
522+
: null,
519523
...(getSecret(character, "COINBASE_API_KEY") &&
520524
getSecret(character, "COINBASE_PRIVATE_KEY")
521525
? [

‎packages/plugin-3d-generation/.npmignore

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
*
2+
3+
!dist/**
4+
!package.json
5+
!readme.md
6+
!tsup.config.ts
7+
!tsconfig.json

‎packages/plugin-3d-generation/eslint.config.mjs

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
import eslintGlobalConfig from "../../eslint.config.mjs";
2+
3+
export default [...eslintGlobalConfig];

‎packages/plugin-3d-generation/package.json

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"name": "@elizaos/plugin-3d-generation",
3+
"version": "0.1.7-alpha.1",
4+
"main": "dist/index.js",
5+
"type": "module",
6+
"types": "dist/index.d.ts",
7+
"dependencies": {
8+
"@elizaos/core": "workspace:*",
9+
"tsup": "8.3.5"
10+
},
11+
"scripts": {
12+
"build": "tsup --format esm --dts",
13+
"dev": "tsup --format esm --dts --watch",
14+
"lint": "eslint --fix --cache ."
15+
},
16+
"peerDependencies": {
17+
"whatwg-url": "7.1.0"
18+
}
19+
}

‎packages/plugin-3d-generation/src/constants.ts

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
export const FAL_CONSTANTS = {
2+
API_3D_ENDPOINT: "fal-ai/hyper3d/rodin",
3+
API_KEY_SETTING: "FAL_API_KEY", // The setting name to fetch from runtime
4+
};

‎packages/plugin-3d-generation/src/index.ts

+198
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
import { elizaLogger } from "@elizaos/core";
2+
import {
3+
Action,
4+
HandlerCallback,
5+
IAgentRuntime,
6+
Memory,
7+
Plugin,
8+
State,
9+
} from "@elizaos/core";
10+
import { fal } from "@fal-ai/client";
11+
import { FAL_CONSTANTS } from "./constants";
12+
13+
import * as fs from 'fs';
14+
import { Buffer } from 'buffer';
15+
import * as path from 'path';
16+
import * as process from 'process';
17+
18+
const generate3D = async (prompt: string, runtime: IAgentRuntime) => {
19+
process.env['FAL_KEY'] = FAL_CONSTANTS.API_KEY_SETTING || runtime.getSetting("FAL_API_KEY");
20+
21+
try {
22+
elizaLogger.log("Starting 3D generation with prompt:", prompt);
23+
24+
const response = await fal.subscribe(FAL_CONSTANTS.API_3D_ENDPOINT, {
25+
input: {
26+
prompt: prompt,
27+
input_image_urls: [],
28+
condition_mode: "concat", // fuse concat
29+
geometry_file_format: "glb", // glb usdz fbx obj stl
30+
material: "PBR", // PBR Shaded
31+
quality: "medium", // extra-low, low, medium, high
32+
tier: "Regular" // Regular, Sketch
33+
},
34+
logs: true,
35+
onQueueUpdate: (update) => {
36+
if (update.status === "IN_PROGRESS") {
37+
update.logs.map((log) => log.message).forEach(elizaLogger.log);
38+
}
39+
},
40+
});
41+
42+
43+
elizaLogger.log(
44+
"Generation request successful, received response:",
45+
response
46+
);
47+
48+
49+
return {success: true,
50+
url: response.data.model_mesh.url,
51+
file_name: response.data.model_mesh.file_name};
52+
53+
} catch (error) {
54+
elizaLogger.error("3D generation error:", error);
55+
return {
56+
success: false,
57+
error: error.message || "Unknown error occurred",
58+
};
59+
}
60+
};
61+
62+
const ThreeDGeneration: Action = {
63+
name: "GENERATE_3D",
64+
similes: [
65+
"3D_GENERATION",
66+
"3D_GEN",
67+
"CREATE_3D",
68+
"MAKE_3D",
69+
"TEXT23D",
70+
"TEXT_TO_3D",
71+
"3D_CREATE",
72+
"3D_MAKE",
73+
],
74+
description: "Generate a 3D object based on a text prompt",
75+
validate: async (runtime: IAgentRuntime, _message: Memory) => {
76+
elizaLogger.log("Validating 3D generation action");
77+
const FalApiKey = runtime.getSetting("FAL_API_KEY");
78+
elizaLogger.log("FAL_API_KEY present:", !!FalApiKey);
79+
return !!FalApiKey;
80+
},
81+
handler: async (
82+
runtime: IAgentRuntime,
83+
message: Memory,
84+
_state: State,
85+
_options: any,
86+
callback: HandlerCallback
87+
) => {
88+
elizaLogger.log("3D generation request:", message);
89+
90+
// Clean up the prompt by removing mentions and commands
91+
const ThreeDPrompt = message.content.text
92+
.replace(/<@\d+>/g, "") // Remove mentions
93+
.replace(
94+
/generate 3D|create 3D|make 3D|render 3D/gi,
95+
""
96+
) // Remove commands
97+
.trim();
98+
99+
if (!ThreeDPrompt || ThreeDPrompt.length < 3) {
100+
callback({
101+
text: "Could you please provide more details about what kind of 3D object you'd like me to generate? For example: 'Generate a lovely cat'",
102+
});
103+
return;
104+
}
105+
106+
elizaLogger.log("3D prompt:", ThreeDPrompt);
107+
108+
callback({
109+
text: `I'll generate a 3D object based on your prompt: "${ThreeDPrompt}". This might take a few minutes...`,
110+
});
111+
112+
try {
113+
const result = await generate3D(ThreeDPrompt, runtime);
114+
115+
if (result.success && result.url && result.file_name) {
116+
// Download the 3D file
117+
const response = await fetch(result.url);
118+
const arrayBuffer = await response.arrayBuffer();
119+
const ThreeDFileName = `content_cache/generated_3d_${result.file_name}`;
120+
121+
// ensure the directory is existed
122+
const directoryPath = path.dirname(ThreeDFileName);
123+
if (!fs.existsSync(directoryPath)) {
124+
fs.mkdirSync(directoryPath, { recursive: true });
125+
}
126+
127+
// Save 3D file
128+
fs.writeFileSync(ThreeDFileName, Buffer.from(arrayBuffer));
129+
130+
callback(
131+
{
132+
text: "Here's your generated 3D object!",
133+
attachments: [
134+
{
135+
id: crypto.randomUUID(),
136+
url: result.url,
137+
title: "Generated 3D",
138+
source: "ThreeDGeneration",
139+
description: ThreeDPrompt,
140+
text: ThreeDPrompt,
141+
},
142+
],
143+
},
144+
[ThreeDFileName]
145+
); // Add the 3D file to the attachments
146+
} else {
147+
callback({
148+
text: `3D generation failed: ${result.error}`,
149+
error: true,
150+
});
151+
}
152+
} catch (error) {
153+
elizaLogger.error(`Failed to generate 3D. Error: ${error}`);
154+
callback({
155+
text: `3D generation failed: ${error.message}`,
156+
error: true,
157+
});
158+
}
159+
},
160+
examples: [
161+
[
162+
{
163+
user: "{{user1}}",
164+
content: { text: "Generate a 3D object of a cat playing piano" },
165+
},
166+
{
167+
user: "{{agentName}}",
168+
content: {
169+
text: "I'll create a 3D object of a cat playing piano for you",
170+
action: "GENERATE_3D",
171+
},
172+
},
173+
],
174+
[
175+
{
176+
user: "{{user1}}",
177+
content: {
178+
text: "Can you make a 3D object of a anime character Goku?",
179+
},
180+
},
181+
{
182+
user: "{{agentName}}",
183+
content: {
184+
text: "I'll generate a 3D object of a anime character Goku for you",
185+
action: "GENERATE_3D",
186+
},
187+
},
188+
],
189+
],
190+
} as Action;
191+
192+
export const ThreeDGenerationPlugin: Plugin = {
193+
name: "3DGeneration",
194+
description: "Generate 3D using Hyper 3D",
195+
actions: [ThreeDGeneration],
196+
evaluators: [],
197+
providers: [],
198+
};

‎packages/plugin-3d-generation/tsconfig.json

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"extends": "../core/tsconfig.json",
3+
"compilerOptions": {
4+
"outDir": "dist",
5+
"rootDir": "src",
6+
"module": "ESNext",
7+
"moduleResolution": "Bundler",
8+
"types": [
9+
"node"
10+
]
11+
},
12+
"include": [
13+
"src/**/*.ts"
14+
]
15+
}

‎packages/plugin-3d-generation/tsup.config.ts

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import { defineConfig } from "tsup";
2+
3+
export default defineConfig({
4+
entry: ["src/index.ts"],
5+
outDir: "dist",
6+
sourcemap: true,
7+
clean: true,
8+
format: ["esm"],
9+
external: [
10+
"dotenv",
11+
"fs",
12+
"path",
13+
"process",
14+
"@reflink/reflink",
15+
"@node-llama-cpp",
16+
"@fal-ai/client",
17+
"https",
18+
"http",
19+
"agentkeepalive",
20+
],
21+
});

0 commit comments

Comments
 (0)