Skip to content

Commit af5508e

Browse files
MarcoMandarMarcoMandar
authored
Merge pull request #16 from ai16z/main
merge from main
2 parents f9733e0 + 2d17b37 commit af5508e

File tree

168 files changed

+11855
-6402
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

168 files changed

+11855
-6402
lines changed

‎.github/workflows/generate-changelog.yml

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: Generate Changelog
2+
on:
3+
push:
4+
tags:
5+
- '*'
6+
jobs:
7+
changelog:
8+
runs-on: ubuntu-latest
9+
permissions:
10+
contents: write
11+
steps:
12+
- uses: actions/checkout@v4
13+
with:
14+
ref: main
15+
token: ${{ secrets.CHANGELOG_GITHUB_TOKEN }}
16+
- name: Generate Changelog
17+
run: |
18+
export PATH="$PATH:/home/runner/.local/share/gem/ruby/3.0.0/bin"
19+
gem install --user-install github_changelog_generator
20+
github_changelog_generator \
21+
-u ${{ github.repository_owner }} \
22+
-p ${{ github.event.repository.name }} \
23+
--token ${{ secrets.CHANGELOG_GITHUB_TOKEN }}
24+
- name: Commit Changelog
25+
uses: stefanzweifel/git-auto-commit-action@v5
26+
with:
27+
commit_message: "chore: update changelog"
28+
branch: main
29+
file_pattern: "CHANGELOG.md"
30+
commit_author: "GitHub Action <actions@github.com>"

‎.npmrc

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
node-linker=hoisted

‎CHANGELOG.md

+199
Large diffs are not rendered by default.

‎core/.env.example

+2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
DISCORD_APPLICATION_ID=
33
DISCORD_API_TOKEN= # Bot token
44
OPENAI_API_KEY=sk-* # OpenAI API key, starting with sk-
5+
GROQ_API_KEY=gsk_*
6+
57
ELEVENLABS_XI_API_KEY= # API key from elevenlabs
68

79
# ELEVENLABS SETTINGS

‎core/package.json

+4-4
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
"type": "module",
77
"types": "dist/index.d.ts",
88
"scripts": {
9-
"build": "tsc",
9+
"build": "tsc -p tsconfig.json",
1010
"lint": "eslint . --fix",
1111
"start": "node --loader ts-node/esm src/index.ts",
1212
"start:arok": "node --loader ts-node/esm src/index.ts --characters=\"../characters/arok.character.json\"",
@@ -20,7 +20,7 @@
2020
"start:degen": "node --loader ts-node/esm src/index.ts --characters=\"../characters/degenspartan.json\"",
2121
"start:service:all": "pm2 start pnpm --name=\"all\" --restart-delay=3000 --max-restarts=10 -- run start:all",
2222
"stop:service:all": "pm2 stop all",
23-
"start:all": "node --loader ts-node/esm src/index.ts --characters=\"../characters/degenspartan.json\",\"../characters/ruby.character.json\",\"../characters/pmairca.character.json\"",
23+
"start:all": "node --loader ts-node/esm src/index.ts --characters=\"../characters/degenspartan.json\",\"../characters/ruby.character.json\",\"../characters/pmairca.json\"",
2424
"start:trump": "node --loader ts-node/esm src/index.ts --characters=\"../characters/trump.character.json\"",
2525
"start:service:tate": "pm2 start pnpm --name=\"tate\" --restart-delay=3000 --max-restarts=10 -- run start:tate",
2626
"stop:service:tate": "pm2 stop tate",
@@ -48,6 +48,7 @@
4848
"@types/fluent-ffmpeg": "2.1.27",
4949
"@types/jest": "29.5.14",
5050
"@types/node": "22.8.4",
51+
"@types/pdfjs-dist": "^2.10.378",
5152
"@types/sql.js": "1.4.9",
5253
"@types/tar": "6.1.13",
5354
"@types/wav-encoder": "1.3.3",
@@ -67,8 +68,7 @@
6768
"ts-node": "10.9.2",
6869
"tslib": "2.8.0",
6970
"typescript": "5.6.3",
70-
"wrangler": "3.84.0",
71-
"@types/pdfjs-dist": "^2.10.378"
71+
"wrangler": "3.84.0"
7272
},
7373
"pnpm": {
7474
"overrides": {

‎core/src/actions/askClaude.ts

+1
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,7 @@ export const askClaude: Action = {
127127

128128
const response = {
129129
userId,
130+
agentId: runtime.agentId,
130131
content: callbackData,
131132
roomId,
132133
embedding: embeddingZeroVector,

‎core/src/actions/imageGeneration.ts

+32-3
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,15 @@ import { prettyConsole } from "../index.ts";
99
import { generateCaption, generateImage } from "./imageGenerationUtils.ts";
1010

1111
export const imageGeneration: Action = {
12-
name: "IMAGE_GEN",
13-
similes: ["GENERATE_IMAGE", "CREATE_IMAGE", "MAKE_PICTURE"],
14-
description: "Generate an image based on a prompt",
12+
name: "GENERATE_IMAGE",
13+
similes: ["IMAGE_GENERATION", "IMAGE_GEN", "CREATE_IMAGE", "MAKE_PICTURE"],
14+
description: "Generate an image to go along with the message.",
1515
validate: async (runtime: IAgentRuntime, message: Memory) => {
1616
const anthropicApiKeyOk = !!runtime.getSetting("ANTHROPIC_API_KEY");
1717
const togetherApiKeyOk = !!runtime.getSetting("TOGETHER_API_KEY");
18+
19+
// TODO: Add openai DALL-E generation as well
20+
1821
return anthropicApiKeyOk && togetherApiKeyOk;
1922
},
2023
handler: async (
@@ -31,6 +34,9 @@ export const imageGeneration: Action = {
3134

3235
const imagePrompt = message.content.text;
3336
prettyConsole.log("Image prompt received:", imagePrompt);
37+
38+
// TODO: Generate a prompt for the image
39+
3440
const res: { image: string; caption: string }[] = [];
3541

3642
prettyConsole.log("Generating image with prompt:", imagePrompt);
@@ -88,35 +94,58 @@ export const imageGeneration: Action = {
8894
}
8995
},
9096
examples: [
97+
98+
// TODO: We want to generate images in more abstract ways, not just when asked to generate an image
99+
91100
[
92101
{
93102
user: "{{user1}}",
94103
content: { text: "Generate an image of a cat" },
95104
},
105+
{
106+
user: "{{agentName}}",
107+
content: { text: "Here's an image of a cat", action: "GENERATE_IMAGE" },
108+
}
96109
],
97110
[
98111
{
99112
user: "{{user1}}",
100113
content: { text: "Generate an image of a dog" },
101114
},
115+
{
116+
user: "{{agentName}}",
117+
content: { text: "Here's an image of a dog", action: "GENERATE_IMAGE" },
118+
}
102119
],
103120
[
104121
{
105122
user: "{{user1}}",
106123
content: { text: "Create an image of a cat with a hat" },
107124
},
125+
{
126+
user: "{{agentName}}",
127+
content: { text: "Here's an image of a cat with a hat", action: "GENERATE_IMAGE" },
128+
}
108129
],
109130
[
110131
{
111132
user: "{{user1}}",
112133
content: { text: "Make an image of a dog with a hat" },
113134
},
135+
{
136+
user: "{{agentName}}",
137+
content: { text: "Here's an image of a dog with a hat", action: "GENERATE_IMAGE" },
138+
}
114139
],
115140
[
116141
{
117142
user: "{{user1}}",
118143
content: { text: "Paint an image of a cat with a hat" },
119144
},
145+
{
146+
user: "{{agentName}}",
147+
content: { text: "Here's an image of a cat with a hat", action: "GENERATE_IMAGE" },
148+
}
120149
],
121150
],
122151
} as Action;

‎core/src/adapters/postgres.ts

+5-5
Original file line numberDiff line numberDiff line change
@@ -269,15 +269,16 @@ export class PostgresDatabaseAdapter extends DatabaseAdapter {
269269

270270
await client.query(
271271
`INSERT INTO memories (
272-
id, type, content, embedding, "userId", "roomId", "unique", "createdAt"
273-
) VALUES ($1, $2, $3, $4::vector, $5::uuid, $6::uuid, $7, to_timestamp($8/1000.0))`,
272+
id, type, content, embedding, "userId", "roomId", "agentId", "unique", "createdAt"
273+
) VALUES ($1, $2, $3, $4::vector, $5::uuid, $6::uuid, $7::uuid, $8, to_timestamp($9/1000.0))`,
274274
[
275275
memory.id ?? v4(),
276276
tableName,
277277
JSON.stringify(memory.content),
278278
`[${memory.embedding.join(",")}]`,
279279
memory.userId,
280280
memory.roomId,
281+
memory.agentId,
281282
memory.unique ?? isUnique,
282283
Date.now(),
283284
]
@@ -365,7 +366,7 @@ export class PostgresDatabaseAdapter extends DatabaseAdapter {
365366
}
366367

367368
if (params.agentId) {
368-
sql += " AND userId = $3";
369+
sql += " AND agentId = $3";
369370
values.push(params.agentId);
370371
}
371372

@@ -638,9 +639,8 @@ export class PostgresDatabaseAdapter extends DatabaseAdapter {
638639
sql += ` AND "unique" = true`;
639640
}
640641

641-
// TODO: Test this
642642
if (params.agentId) {
643-
sql += " AND userId = $3";
643+
sql += " AND agentId = $3";
644644
values.push(params.agentId);
645645
}
646646

‎core/src/adapters/sqlite.ts

+11-5
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ export class SqliteDatabaseAdapter extends DatabaseAdapter {
162162
let queryParams = [params.tableName, ...params.roomIds];
163163

164164
if (params.agentId) {
165-
sql += ` AND userId = ?`;
165+
sql += ` AND agentId = ?`;
166166
queryParams.push(params.agentId);
167167
}
168168

@@ -219,14 +219,15 @@ export class SqliteDatabaseAdapter extends DatabaseAdapter {
219219
const createdAt = memory.createdAt ?? Date.now();
220220

221221
// Insert the memory with the appropriate 'unique' value
222-
const sql = `INSERT OR REPLACE INTO memories (id, type, content, embedding, userId, roomId, \`unique\`, createdAt) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`;
222+
const sql = `INSERT OR REPLACE INTO memories (id, type, content, embedding, userId, roomId, agentId, \`unique\`, createdAt) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`;
223223
this.db.prepare(sql).run(
224224
memory.id ?? v4(),
225225
tableName,
226226
content,
227227
new Float32Array(memory.embedding ?? embeddingZeroVector), // Store as Float32Array
228228
memory.userId,
229229
memory.roomId,
230+
memory.agentId,
230231
isUnique ? 1 : 0,
231232
createdAt
232233
);
@@ -235,6 +236,7 @@ export class SqliteDatabaseAdapter extends DatabaseAdapter {
235236
async searchMemories(params: {
236237
tableName: string;
237238
roomId: UUID;
239+
agentId?: UUID;
238240
embedding: number[];
239241
match_threshold: number;
240242
match_count: number;
@@ -256,6 +258,11 @@ export class SqliteDatabaseAdapter extends DatabaseAdapter {
256258
sql += " AND `unique` = 1";
257259
}
258260

261+
if (params.agentId) {
262+
sql += " AND agentId = ?";
263+
queryParams.push(params.agentId);
264+
}
265+
259266
sql += ` ORDER BY similarity ASC LIMIT ?`; // ASC for lower distance
260267
// Updated queryParams order matches the placeholders
261268

@@ -297,9 +304,8 @@ export class SqliteDatabaseAdapter extends DatabaseAdapter {
297304
if (params.unique) {
298305
sql += " AND `unique` = 1";
299306
}
300-
// TODO: Test this
301307
if (params.agentId) {
302-
sql += " AND userId = ?";
308+
sql += " AND agentId = ?";
303309
queryParams.push(params.agentId);
304310
}
305311

@@ -413,7 +419,7 @@ export class SqliteDatabaseAdapter extends DatabaseAdapter {
413419
}
414420

415421
if (params.agentId) {
416-
sql += " AND userId = ?";
422+
sql += " AND agentId = ?";
417423
queryParams.push(params.agentId);
418424
}
419425

‎core/src/adapters/sqljs.ts

+3-2
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ export class SqlJsDatabaseAdapter extends DatabaseAdapter {
236236
}
237237

238238
// Insert the memory with the appropriate 'unique' value
239-
const sql = `INSERT INTO memories (id, type, content, embedding, userId, roomId, \`unique\`, createdAt) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`;
239+
const sql = `INSERT INTO memories (id, type, content, embedding, userId, roomId, agentId, \`unique\`, createdAt) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`;
240240
const stmt = this.db.prepare(sql);
241241

242242
const createdAt = memory.createdAt ?? Date.now();
@@ -248,6 +248,7 @@ export class SqlJsDatabaseAdapter extends DatabaseAdapter {
248248
JSON.stringify(memory.embedding),
249249
memory.userId,
250250
memory.roomId,
251+
memory.agentId,
251252
isUnique ? 1 : 0,
252253
createdAt,
253254
]);
@@ -461,7 +462,7 @@ export class SqlJsDatabaseAdapter extends DatabaseAdapter {
461462
}
462463

463464
if (params.agentId) {
464-
sql += " AND userId = ?";
465+
sql += " AND agentId = ?";
465466
}
466467

467468
sql += " ORDER BY createdAt DESC";

‎core/src/clients/direct/index.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -162,11 +162,11 @@ class DirectClient {
162162
inReplyTo: undefined,
163163
};
164164

165-
const userMessage = { content, userId, roomId };
165+
const userMessage = { content, userId, roomId, agentId: runtime.agentId };
166166

167167
const memory: Memory = {
168168
id: messageId,
169-
...userMessage,
169+
agentId: runtime.agentId,
170170
userId,
171171
roomId,
172172
content,

‎core/src/clients/discord/index.ts

+9-7
Original file line numberDiff line numberDiff line change
@@ -168,12 +168,12 @@ export class DiscordClient extends EventEmitter {
168168

169169
const reactionMessage = `*<${emoji}>: "${truncatedContent}"*`;
170170

171-
const roomId = stringToUuid(reaction.message.channel.id);
172-
const userIdUUID = stringToUuid(user.id);
171+
const roomId = stringToUuid(reaction.message.channel.id + "-" + this.runtime.agentId);
172+
const userIdUUID = stringToUuid(user.id + "-" + this.runtime.agentId);
173173

174174
// Generate a unique UUID for the reaction
175175
const reactionUUID = stringToUuid(
176-
`${reaction.message.id}-${user.id}-${emoji}`
176+
`${reaction.message.id}-${user.id}-${emoji}-${this.runtime.agentId}`
177177
);
178178

179179
// ensure the user id and room id are valid
@@ -196,10 +196,11 @@ export class DiscordClient extends EventEmitter {
196196
await this.runtime.messageManager.createMemory({
197197
id: reactionUUID, // This is the ID of the reaction message
198198
userId: userIdUUID,
199+
agentId: this.runtime.agentId,
199200
content: {
200201
text: reactionMessage,
201202
source: "discord",
202-
inReplyTo: stringToUuid(reaction.message.id), // This is the ID of the original message
203+
inReplyTo: stringToUuid(reaction.message.id + "-" + this.runtime.agentId), // This is the ID of the original message
203204
},
204205
roomId,
205206
createdAt: Date.now(),
@@ -237,12 +238,12 @@ export class DiscordClient extends EventEmitter {
237238

238239
const reactionMessage = `*Removed <${emoji} emoji> from: "${truncatedContent}"*`;
239240

240-
const roomId = stringToUuid(reaction.message.channel.id);
241+
const roomId = stringToUuid(reaction.message.channel.id + "-" + this.runtime.agentId);
241242
const userIdUUID = stringToUuid(user.id);
242243

243244
// Generate a unique UUID for the reaction removal
244245
const reactionUUID = stringToUuid(
245-
`${reaction.message.id}-${user.id}-${emoji}-removed`
246+
`${reaction.message.id}-${user.id}-${emoji}-removed-${this.runtime.agentId}`
246247
);
247248

248249
const userName = reaction.message.author.username;
@@ -261,10 +262,11 @@ export class DiscordClient extends EventEmitter {
261262
await this.runtime.messageManager.createMemory({
262263
id: reactionUUID, // This is the ID of the reaction removal message
263264
userId: userIdUUID,
265+
agentId: this.runtime.agentId,
264266
content: {
265267
text: reactionMessage,
266268
source: "discord",
267-
inReplyTo: stringToUuid(reaction.message.id), // This is the ID of the original message
269+
inReplyTo: stringToUuid(reaction.message.id + "-" + this.runtime.agentId), // This is the ID of the original message
268270
},
269271
roomId,
270272
createdAt: Date.now(),

0 commit comments

Comments
 (0)