Skip to content

Commit 911984a

Browse files
authored
Merge pull request #1525 from Titan-Node/add-livepeer-image-gen-1
feat: Add Livepeer Image Provider
2 parents 438d736 + f8028c1 commit 911984a

File tree

12 files changed

+123
-5
lines changed

12 files changed

+123
-5
lines changed

.env.example

+4
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@ SMALL_HYPERBOLIC_MODEL= # Default: meta-llama/Llama-3.2-3B-Instruct
3636
MEDIUM_HYPERBOLIC_MODEL= # Default: meta-llama/Meta-Llama-3.1-70B-Instruct
3737
LARGE_HYPERBOLIC_MODEL= # Default: meta-llama/Meta-Llama-3.1-405-Instruct
3838

39+
# Livepeer configuration
40+
LIVEPEER_GATEWAY_URL= # Free inference gateways and docs: https://livepeer-eliza.com/
41+
LIVEPEER_IMAGE_MODEL= # Default: ByteDance/SDXL-Lightning
42+
3943
# Speech Synthesis
4044
ELEVENLABS_XI_API_KEY= # API key from elevenlabs
4145

agent/src/index.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -554,7 +554,8 @@ export async function createAgent(
554554
getSecret(character, "FAL_API_KEY") ||
555555
getSecret(character, "OPENAI_API_KEY") ||
556556
getSecret(character, "VENICE_API_KEY") ||
557-
getSecret(character, "HEURIST_API_KEY")
557+
getSecret(character, "HEURIST_API_KEY") ||
558+
getSecret(character, "LIVEPEER_GATEWAY_URL")
558559
? imageGenerationPlugin
559560
: null,
560561
getSecret(character, "FAL_API_KEY") ? ThreeDGenerationPlugin : null,

docs/api/enumerations/ModelProviderName.md

+10
Original file line numberDiff line numberDiff line change
@@ -233,3 +233,13 @@ Available model providers
233233
#### Defined in
234234

235235
[packages/core/src/types.ts:240](https://github.com/elizaOS/eliza/blob/main/packages/core/src/types.ts#L240)
236+
237+
***
238+
239+
### LIVEPEER
240+
241+
> **LIVEPEER**: `"livepeer"`
242+
243+
#### Defined in
244+
245+
[packages/core/src/types.ts:241](https://github.com/elizaOS/eliza/blob/main/packages/core/src/types.ts#L241)

docs/api/type-aliases/Models.md

+4
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,10 @@ Model configurations by provider
100100

101101
> **akash\_chat\_api**: [`Model`](Model.md)
102102
103+
### livepeer
104+
105+
> **livepeer**: [`Model`](Model.md)
106+
103107
## Defined in
104108

105109
[packages/core/src/types.ts:188](https://github.com/elizaOS/eliza/blob/main/packages/core/src/types.ts#L188)

docs/docs/api/enumerations/ModelProviderName.md

+10
Original file line numberDiff line numberDiff line change
@@ -119,3 +119,13 @@
119119
#### Defined in
120120

121121
[packages/core/src/types.ts:132](https://github.com/elizaos/eliza/blob/4d1e66cbf7deea87a8a67525670a963cd00108bc/packages/core/src/types.ts#L132)
122+
123+
---
124+
125+
### LIVEPEER
126+
127+
> **LIVEPEER**: `"livepeer"`
128+
129+
#### Defined in
130+
131+
[packages/core/src/types.ts:133](https://github.com/elizaos/eliza/blob/4d1e66cbf7deea87a8a67525670a963cd00108bc/packages/core/src/types.ts#L133)

docs/docs/api/type-aliases/Models.md

+4
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,10 @@
5252

5353
> **heurist**: [`Model`](Model.md)
5454
55+
### livepeer
56+
57+
> **livepeer**: [`Model`](Model.md)
58+
5559
## Defined in
5660

5761
[packages/core/src/types.ts:105](https://github.com/elizaos/eliza/blob/7fcf54e7fb2ba027d110afcc319c0b01b3f181dc/packages/core/src/types.ts#L105)

docs/docs/guides/configuration.md

+3
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,9 @@ TOGETHER_API_KEY=
7272
# Heurist Settings
7373
HEURIST_API_KEY=
7474

75+
# Livepeer Settings
76+
LIVEPEER_GATEWAY_URL=
77+
7578
# Local Model Settings
7679
XAI_MODEL=meta-llama/Llama-3.1-7b-instruct
7780
```

docs/docs/quickstart.md

+4-2
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ pnpm build
8282
OPENAI_API_KEY= # OpenAI API key
8383
GROK_API_KEY= # Grok API key
8484
ELEVENLABS_XI_API_KEY= # API key from elevenlabs (for voice)
85+
LIVEPEER_GATEWAY_URL= # Livepeer gateway URL
8586
```
8687
8788
## Choose Your Model
@@ -94,6 +95,7 @@ Eliza supports multiple AI models:
9495
- **Llama**: Set `XAI_MODEL=meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo`
9596
- **Grok**: Set `XAI_MODEL=grok-beta`
9697
- **OpenAI**: Set `XAI_MODEL=gpt-4o-mini` or `gpt-4o`
98+
- **Livepeer**: Set `LIVEPEER_IMAGE_MODEL` to your chosen Livepeer image model, available models [here](https://livepeer-eliza.com/)
9799
98100
You set which model to use inside the character JSON file
99101
@@ -216,8 +218,8 @@ pnpm start --characters="characters/trump.character.json,characters/tate.charact
216218
- Ensure Node.js 23.3.0 is installed
217219
- Use `node -v` to check version
218220
- Consider using [nvm](https://github.com/nvm-sh/nvm) to manage Node versions
219-
220-
NOTE: pnpm may be bundled with a different node version, ignoring nvm. If this is the case, you can use
221+
222+
NOTE: pnpm may be bundled with a different node version, ignoring nvm. If this is the case, you can use
221223
```bash
222224
pnpm env use --global 23.3.0
223225
```

packages/core/src/generation.ts

+60-1
Original file line numberDiff line numberDiff line change
@@ -978,13 +978,16 @@ export const generateImage = async (
978978
return runtime.getSetting("OPENAI_API_KEY");
979979
case ModelProviderName.VENICE:
980980
return runtime.getSetting("VENICE_API_KEY");
981+
case ModelProviderName.LIVEPEER:
982+
return runtime.getSetting("LIVEPEER_GATEWAY_URL");
981983
default:
982984
// If no specific match, try the fallback chain
983985
return (runtime.getSetting("HEURIST_API_KEY") ??
984986
runtime.getSetting("TOGETHER_API_KEY") ??
985987
runtime.getSetting("FAL_API_KEY") ??
986988
runtime.getSetting("OPENAI_API_KEY") ??
987-
runtime.getSetting("VENICE_API_KEY"));
989+
runtime.getSetting("VENICE_API_KEY"))??
990+
runtime.getSetting("LIVEPEER_GATEWAY_URL");
988991
}
989992
})();
990993
try {
@@ -1176,6 +1179,62 @@ export const generateImage = async (
11761179
});
11771180

11781181
return { success: true, data: base64s };
1182+
1183+
} else if (runtime.imageModelProvider === ModelProviderName.LIVEPEER) {
1184+
if (!apiKey) {
1185+
throw new Error("Livepeer Gateway is not defined");
1186+
}
1187+
try {
1188+
const baseUrl = new URL(apiKey);
1189+
if (!baseUrl.protocol.startsWith('http')) {
1190+
throw new Error("Invalid Livepeer Gateway URL protocol");
1191+
}
1192+
const response = await fetch(`${baseUrl.toString()}text-to-image`, {
1193+
method: "POST",
1194+
headers: {
1195+
"Content-Type": "application/json"
1196+
},
1197+
body: JSON.stringify({
1198+
model_id: data.modelId || "ByteDance/SDXL-Lightning",
1199+
prompt: data.prompt,
1200+
width: data.width || 1024,
1201+
height: data.height || 1024
1202+
})
1203+
});
1204+
const result = await response.json();
1205+
if (!result.images?.length) {
1206+
throw new Error("No images generated");
1207+
}
1208+
const base64Images = await Promise.all(
1209+
result.images.map(async (image) => {
1210+
console.log("imageUrl console log", image.url);
1211+
let imageUrl;
1212+
if (image.url.includes("http")) {
1213+
imageUrl = image.url;
1214+
} else {
1215+
imageUrl = `${apiKey}${image.url}`;
1216+
}
1217+
const imageResponse = await fetch(imageUrl);
1218+
if (!imageResponse.ok) {
1219+
throw new Error(
1220+
`Failed to fetch image: ${imageResponse.statusText}`
1221+
);
1222+
}
1223+
const blob = await imageResponse.blob();
1224+
const arrayBuffer = await blob.arrayBuffer();
1225+
const base64 = Buffer.from(arrayBuffer).toString("base64");
1226+
return `data:image/jpeg;base64,${base64}`;
1227+
})
1228+
);
1229+
return {
1230+
success: true,
1231+
data: base64Images
1232+
};
1233+
} catch (error) {
1234+
console.error(error);
1235+
return { success: false, error: error };
1236+
}
1237+
11791238
} else {
11801239
let targetSize = `${data.width}x${data.height}`;
11811240
if (

packages/core/src/models.ts

+17
Original file line numberDiff line numberDiff line change
@@ -486,6 +486,23 @@ export const models: Models = {
486486
"Meta-Llama-3-1-405B-Instruct-FP8",
487487
},
488488
},
489+
[ModelProviderName.LIVEPEER]: {
490+
settings: {
491+
stop: [],
492+
maxInputTokens: 128000,
493+
maxOutputTokens: 8192,
494+
repetition_penalty: 0.4,
495+
temperature: 0.7,
496+
},
497+
// livepeer endpoint is handled from the sdk
498+
model: {
499+
[ModelClass.SMALL]: "",
500+
[ModelClass.MEDIUM]: "",
501+
[ModelClass.LARGE]: "",
502+
[ModelClass.EMBEDDING]: "",
503+
[ModelClass.IMAGE]: settings.LIVEPEER_IMAGE_MODEL || "ByteDance/SDXL-Lightning",
504+
},
505+
},
489506
};
490507

491508
export function getModel(provider: ModelProviderName, type: ModelClass) {

packages/core/src/types.ts

+2
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,7 @@ export type Models = {
209209
[ModelProviderName.HYPERBOLIC]: Model;
210210
[ModelProviderName.VENICE]: Model;
211211
[ModelProviderName.AKASH_CHAT_API]: Model;
212+
[ModelProviderName.LIVEPEER]: Model;
212213
};
213214

214215
/**
@@ -238,6 +239,7 @@ export enum ModelProviderName {
238239
HYPERBOLIC = "hyperbolic",
239240
VENICE = "venice",
240241
AKASH_CHAT_API = "akash_chat_api",
242+
LIVEPEER = "livepeer",
241243
}
242244

243245
/**

packages/plugin-image-generation/src/index.ts

+3-1
Original file line numberDiff line numberDiff line change
@@ -86,14 +86,16 @@ const imageGeneration: Action = {
8686
const falApiKeyOk = !!runtime.getSetting("FAL_API_KEY");
8787
const openAiApiKeyOk = !!runtime.getSetting("OPENAI_API_KEY");
8888
const veniceApiKeyOk = !!runtime.getSetting("VENICE_API_KEY");
89+
const livepeerGatewayUrlOk = !!runtime.getSetting("LIVEPEER_GATEWAY_URL");
8990

9091
return (
9192
anthropicApiKeyOk ||
9293
togetherApiKeyOk ||
9394
heuristApiKeyOk ||
9495
falApiKeyOk ||
9596
openAiApiKeyOk ||
96-
veniceApiKeyOk
97+
veniceApiKeyOk ||
98+
livepeerGatewayUrlOk
9799
);
98100
},
99101
handler: async (

0 commit comments

Comments
 (0)