Skip to content

Commit ee5422d

Browse files
committed
fix paths and character path
1 parent 671be77 commit ee5422d

File tree

9 files changed

+14869
-10472
lines changed

9 files changed

+14869
-10472
lines changed

core/package.json

+2
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@
8585
"@cliqz/adblocker-playwright": "1.34.0",
8686
"@coral-xyz/anchor": "^0.30.1",
8787
"@discordjs/rest": "2.4.0",
88+
"@discordjs/opus": "github:discordjs/opus",
8889
"@discordjs/voice": "0.17.0",
8990
"@echogarden/espeak-ng-emscripten": "0.3.0",
9091
"@echogarden/kissfft-wasm": "0.2.0",
@@ -175,6 +176,7 @@
175176
"trustedDependencies": {
176177
"onnxruntime-node": "^1.19.2",
177178
"@discordjs/opus": "github:discordjs/opus",
179+
"@discordjs/voice": "0.17.0",
178180
"sharp": "^0.33.5"
179181
},
180182
"peerDependencies": {

core/src/actions/swap.ts

+16-16
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,30 @@
1+
import { bs58 } from "@coral-xyz/anchor/dist/cjs/utils/bytes/index.js";
12
import {
23
Connection,
34
Keypair,
45
PublicKey,
5-
Transaction,
6-
VersionedTransaction,
6+
VersionedTransaction
77
} from "@solana/web3.js";
8+
import BigNumber from "bignumber.js";
89
import fetch from "cross-fetch";
10+
import { v4 as uuidv4 } from "uuid";
11+
import { TrustScoreDatabase } from "../adapters/trustScoreDatabase.ts";
12+
import { composeContext } from "../core/context.ts";
13+
import { generateObject } from "../core/generation.ts";
14+
import settings from "../core/settings.ts";
915
import {
1016
ActionExample,
17+
HandlerCallback,
1118
IAgentRuntime,
1219
Memory,
13-
type Action,
14-
State,
1520
ModelClass,
16-
HandlerCallback,
21+
State,
22+
type Action,
1723
} from "../core/types.ts";
18-
import { walletProvider } from "../providers/wallet.ts";
19-
import { composeContext } from "../core/context.ts";
20-
import { generateObject, generateObjectArray } from "../core/generation.ts";
24+
import { TokenProvider } from "../providers/token.ts";
25+
import { TrustScoreProvider } from "../providers/trustScoreProvider.ts";
26+
import { walletProvider, WalletProvider } from "../providers/wallet.ts";
2127
import { getTokenDecimals } from "./swapUtils.ts";
22-
import settings from "../core/settings.ts";
23-
import { bs58 } from "@coral-xyz/anchor/dist/cjs/utils/bytes/index.js";
24-
import BigNumber from "bignumber.js";
25-
import { WalletProvider } from "../providers/wallet.ts";
26-
import { TrustScoreProvider } from "../providers/trustScoreProvider";
27-
import { TokenProvider } from "../providers/token";
28-
import { TrustScoreDatabase } from "../adapters/trustScoreDatabase";
29-
import { v4 as uuidv4 } from "uuid";
3028

3129
async function swapToken(
3230
connection: Connection,
@@ -418,6 +416,7 @@ export const executeSwap: Action = {
418416
is_simulation: false,
419417
};
420418
await trustScoreDatabase.createTradePerformance(
419+
runtime,
421420
response.outputTokenCA,
422421
recommender.id,
423422
tradeData
@@ -448,6 +447,7 @@ export const executeSwap: Action = {
448447
};
449448
const sellTimeStamp = new Date().getTime().toString();
450449
await trustScoreDatabase.updateSellDetails(
450+
runtime,
451451
response.inputTokenCA,
452452
recommender.id,
453453
sellTimeStamp,

core/src/cli/index.ts

+11-1
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,17 @@ export function parseArguments(): Arguments {
6666
}
6767

6868
export function loadCharacters(charactersArg: string): Character[] {
69-
const characterPaths = charactersArg?.split(",").map((path) => path.trim());
69+
let characterPaths = charactersArg
70+
?.split(",")
71+
.map((path) => path.trim())
72+
.map((path) => {
73+
if (path.startsWith("./characters")) {
74+
return `../characters/${path}`;
75+
}
76+
return path;
77+
});
78+
79+
7080
const loadedCharacters = [];
7181

7282
if (characterPaths?.length > 0) {

core/src/clients/chat/index.ts

+274
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,274 @@
1+
import bodyParser from "body-parser";
2+
import express from "express";
3+
import { composeContext } from "../../core/context.ts";
4+
import { AgentRuntime } from "../../core/runtime.ts";
5+
import { Content, Memory, ModelClass, State } from "../../core/types.ts";
6+
import { stringToUuid } from "../../core/uuid.ts";
7+
import cors from "cors";
8+
import { messageCompletionFooter } from "../../core/parsing.ts";
9+
import multer, { File } from "multer";
10+
import { Request as ExpressRequest } from "express";
11+
import { generateMessageResponse } from "../../core/generation.ts";
12+
import {
13+
generateCaption,
14+
generateImage,
15+
} from "../../actions/imageGenerationUtils.ts";
16+
17+
const upload = multer({ storage: multer.memoryStorage() });
18+
19+
export const messageHandlerTemplate =
20+
// {{goals}}
21+
`# Action Examples
22+
{{actionExamples}}
23+
(Action examples are for reference only. Do not use the information from them in your response.)
24+
25+
# Task: Generate dialog and actions for the character {{agentName}}.
26+
About {{agentName}}:
27+
{{bio}}
28+
{{lore}}
29+
30+
{{providers}}
31+
32+
{{attachments}}
33+
34+
# Capabilities
35+
Note that {{agentName}} is capable of reading/seeing/hearing various forms of media, including images, videos, audio, plaintext and PDFs. Recent attachments have been included above under the "Attachments" section.
36+
37+
{{messageDirections}}
38+
39+
{{recentMessages}}
40+
41+
{{actions}}
42+
43+
# Instructions: Write the next message for {{agentName}}. Ignore "action".
44+
` + messageCompletionFooter;
45+
46+
export interface SimliClientConfig {
47+
apiKey: string;
48+
faceID: string;
49+
handleSilence: boolean;
50+
videoRef: any;
51+
audioRef: any;
52+
}
53+
class DirectClient {
54+
private app: express.Application;
55+
private agents: Map<string, AgentRuntime>;
56+
57+
constructor() {
58+
this.app = express();
59+
this.app.use(cors());
60+
this.agents = new Map();
61+
62+
this.app.use(bodyParser.json());
63+
this.app.use(bodyParser.urlencoded({ extended: true }));
64+
65+
// Define an interface that extends the Express Request interface
66+
interface CustomRequest extends ExpressRequest {
67+
file: File;
68+
}
69+
70+
// Update the route handler to use CustomRequest instead of express.Request
71+
this.app.post(
72+
"/:agentId/whisper",
73+
upload.single("file"),
74+
async (req: CustomRequest, res: express.Response) => {
75+
const audioFile = req.file; // Access the uploaded file using req.file
76+
const agentId = req.params.agentId;
77+
78+
if (!audioFile) {
79+
res.status(400).send("No audio file provided");
80+
return;
81+
}
82+
83+
let runtime = this.agents.get(agentId);
84+
85+
// if runtime is null, look for runtime with the same name
86+
if (!runtime) {
87+
runtime = Array.from(this.agents.values()).find(
88+
(a) =>
89+
a.character.name.toLowerCase() ===
90+
agentId.toLowerCase()
91+
);
92+
}
93+
94+
if (!runtime) {
95+
res.status(404).send("Agent not found");
96+
return;
97+
}
98+
99+
const formData = new FormData();
100+
const audioBlob = new Blob([audioFile.buffer], {
101+
type: audioFile.mimetype,
102+
});
103+
formData.append("file", audioBlob, audioFile.originalname);
104+
formData.append("model", "whisper-1");
105+
106+
const response = await fetch(
107+
"https://api.openai.com/v1/audio/transcriptions",
108+
{
109+
method: "POST",
110+
headers: {
111+
Authorization: `Bearer ${runtime.token}`,
112+
},
113+
body: formData,
114+
}
115+
);
116+
117+
const data = await response.json();
118+
res.json(data);
119+
}
120+
);
121+
122+
this.app.post(
123+
"/:agentId/message",
124+
async (req: express.Request, res: express.Response) => {
125+
const agentId = req.params.agentId;
126+
const roomId = stringToUuid(
127+
req.body.roomId ?? "default-room-" + agentId
128+
);
129+
const userId = stringToUuid(req.body.userId ?? "user");
130+
131+
let runtime = this.agents.get(agentId);
132+
133+
// if runtime is null, look for runtime with the same name
134+
if (!runtime) {
135+
runtime = Array.from(this.agents.values()).find(
136+
(a) =>
137+
a.character.name.toLowerCase() ===
138+
agentId.toLowerCase()
139+
);
140+
}
141+
142+
if (!runtime) {
143+
res.status(404).send("Agent not found");
144+
return;
145+
}
146+
147+
await runtime.ensureConnection(
148+
userId,
149+
roomId,
150+
req.body.userName,
151+
req.body.name,
152+
"direct"
153+
);
154+
155+
const text = req.body.text;
156+
const messageId = stringToUuid(Date.now().toString());
157+
158+
const content: Content = {
159+
text,
160+
attachments: [],
161+
source: "direct",
162+
inReplyTo: undefined,
163+
};
164+
165+
const userMessage = { content, userId, roomId, agentId: runtime.agentId };
166+
167+
const memory: Memory = {
168+
id: messageId,
169+
agentId: runtime.agentId,
170+
userId,
171+
roomId,
172+
content,
173+
createdAt: Date.now(),
174+
};
175+
176+
await runtime.messageManager.createMemory(memory);
177+
178+
const state = (await runtime.composeState(userMessage, {
179+
agentName: runtime.character.name,
180+
})) as State;
181+
182+
const context = composeContext({
183+
state,
184+
template: messageHandlerTemplate,
185+
});
186+
187+
const response = await generateMessageResponse({
188+
runtime: runtime,
189+
context,
190+
modelClass: ModelClass.SMALL,
191+
});
192+
193+
// save response to memory
194+
const responseMessage = {
195+
...userMessage,
196+
userId: runtime.agentId,
197+
content: response,
198+
};
199+
200+
await runtime.messageManager.createMemory(responseMessage);
201+
202+
if (!response) {
203+
res.status(500).send(
204+
"No response from generateMessageResponse"
205+
);
206+
return;
207+
}
208+
209+
let message = null as Content | null;
210+
211+
const result = await runtime.processActions(
212+
memory,
213+
[responseMessage],
214+
state,
215+
async (newMessages) => {
216+
message = newMessages;
217+
return [memory];
218+
}
219+
)
220+
221+
if (message) {
222+
res.json([message, response]);
223+
} else {
224+
res.json([response]);
225+
}
226+
227+
}
228+
);
229+
230+
this.app.post(
231+
"/:agentId/image",
232+
async (req: express.Request, res: express.Response) => {
233+
const agentId = req.params.agentId;
234+
const agent = this.agents.get(agentId);
235+
if (!agent) {
236+
res.status(404).send("Agent not found");
237+
return;
238+
}
239+
240+
const images = await generateImage({ ...req.body }, agent);
241+
const imagesRes: { image: string; caption: string }[] = [];
242+
if (images.data && images.data.length > 0) {
243+
for (let i = 0; i < images.data.length; i++) {
244+
const caption = await generateCaption(
245+
{ imageUrl: images.data[i] },
246+
agent
247+
);
248+
imagesRes.push({
249+
image: images.data[i],
250+
caption: caption.title,
251+
});
252+
}
253+
}
254+
res.json({ images: imagesRes });
255+
}
256+
);
257+
}
258+
259+
public registerAgent(runtime: AgentRuntime) {
260+
this.agents.set(runtime.agentId, runtime);
261+
}
262+
263+
public unregisterAgent(runtime: AgentRuntime) {
264+
this.agents.delete(runtime.agentId);
265+
}
266+
267+
public start(port: number) {
268+
this.app.listen(port, () => {
269+
console.log(`Server running at http://localhost:${port}/`);
270+
});
271+
}
272+
}
273+
274+
export { DirectClient };

core/src/index.ts

+11-1
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,18 @@ import {
2222
import { PrettyConsole } from "./cli/colors.ts";
2323

2424
let argv: Arguments = parseArguments();
25+
let basePath = "./";
26+
// if argv.isroot is true, then set the base path to "../"
27+
if (argv.isRoot) {
28+
basePath = "../";
29+
}
30+
31+
// if the path doesnt start with a /, add the base path to the beginning of the path
32+
if (!argv.characters?.startsWith("/")) {
33+
argv.characters = `${basePath}${argv.characters}`;
34+
}
2535

26-
const characters = loadCharacters(argv.characters);
36+
let characters = loadCharacters(argv.characters);
2737

2838
const directClient = new Client.DirectClient();
2939

0 commit comments

Comments
 (0)