Skip to content

Commit 7e8a627

Browse files
yodamaster726yodamaster726
committed
fix: llama vs ollama issues
1 parent 9330697 commit 7e8a627

File tree

4 files changed

+156
-65
lines changed

4 files changed

+156
-65
lines changed

‎packages/core/src/embedding.ts

+3-3
Original file line numberDiff line numberDiff line change
@@ -87,9 +87,9 @@ export async function embed(runtime: IAgentRuntime, input: string) {
8787
const embeddingModel = settings.USE_OPENAI_EMBEDDING
8888
? "text-embedding-3-small"
8989
: runtime.character.modelProvider === ModelProviderName.OLLAMA
90-
? settings.OLLAMA_EMBEDDING_MODEL || "mxbai-embed-large"
91-
: modelProvider.model?.[ModelClass.EMBEDDING]||
92-
models[ModelProviderName.OPENAI].model[ModelClass.EMBEDDING];;
90+
? settings.OLLAMA_EMBEDDING_MODEL || "mxbai-embed-large"
91+
: modelProvider.model?.[ModelClass.EMBEDDING] ||
92+
models[ModelProviderName.OPENAI].model[ModelClass.EMBEDDING];
9393

9494
if (!embeddingModel) {
9595
throw new Error("No embedding model configured");

‎packages/core/src/runtime.ts

+11-5
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ export class AgentRuntime implements IAgentRuntime {
222222
elizaLogger.info("Initializing AgentRuntime with options:", {
223223
character: opts.character?.name,
224224
modelProvider: opts.modelProvider,
225-
characterModelProvider: opts.character?.modelProvider
225+
characterModelProvider: opts.character?.modelProvider,
226226
});
227227

228228
this.#conversationLength =
@@ -288,23 +288,29 @@ export class AgentRuntime implements IAgentRuntime {
288288
});
289289

290290
this.serverUrl = opts.serverUrl ?? this.serverUrl;
291-
291+
292292
elizaLogger.info("Setting model provider...");
293-
elizaLogger.info("- Character model provider:", this.character.modelProvider);
293+
elizaLogger.info(
294+
"- Character model provider:",
295+
this.character.modelProvider
296+
);
294297
elizaLogger.info("- Opts model provider:", opts.modelProvider);
295298
elizaLogger.info("- Current model provider:", this.modelProvider);
296299

297300
this.modelProvider =
298301
this.character.modelProvider ??
299302
opts.modelProvider ??
300303
this.modelProvider;
301-
304+
302305
elizaLogger.info("Selected model provider:", this.modelProvider);
303306

304307
// Validate model provider
305308
if (!Object.values(ModelProviderName).includes(this.modelProvider)) {
306309
elizaLogger.error("Invalid model provider:", this.modelProvider);
307-
elizaLogger.error("Available providers:", Object.values(ModelProviderName));
310+
elizaLogger.error(
311+
"Available providers:",
312+
Object.values(ModelProviderName)
313+
);
308314
throw new Error(`Invalid model provider: ${this.modelProvider}`);
309315
}
310316

‎packages/plugin-node/src/services/image.ts

+10-3
Original file line numberDiff line numberDiff line change
@@ -71,9 +71,16 @@ export class ImageDescriptionService
7171
device: "gpu",
7272
progress_callback: (progress) => {
7373
if (progress.status === "downloading") {
74-
const percent = ((progress.loaded / progress.total) * 100).toFixed(1);
75-
const dots = '.'.repeat(Math.floor(Number(percent) / 5));
76-
elizaLogger.info(`Downloading Florence model: [${dots.padEnd(20, ' ')}] ${percent}%`);
74+
const percent = (
75+
(progress.loaded / progress.total) *
76+
100
77+
).toFixed(1);
78+
const dots = ".".repeat(
79+
Math.floor(Number(percent) / 5)
80+
);
81+
elizaLogger.info(
82+
`Downloading Florence model: [${dots.padEnd(20, " ")}] ${percent}%`
83+
);
7784
}
7885
},
7986
}

‎packages/plugin-node/src/services/llama.ts

+132-54
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
1-
import { elizaLogger, IAgentRuntime, ServiceType, ModelProviderName } from "@ai16z/eliza";
1+
import {
2+
elizaLogger,
3+
IAgentRuntime,
4+
ServiceType,
5+
ModelProviderName,
6+
} from "@ai16z/eliza";
27
import { Service } from "@ai16z/eliza";
38
import fs from "fs";
49
import https from "https";
@@ -198,13 +203,17 @@ export class LlamaService extends Service {
198203
} catch (error) {
199204
elizaLogger.error("Failed to initialize LlamaService:", error);
200205
// Re-throw with more context
201-
throw new Error(`LlamaService initialization failed: ${error.message}`);
206+
throw new Error(
207+
`LlamaService initialization failed: ${error.message}`
208+
);
202209
}
203210
}
204211

205212
private async ensureInitialized() {
206213
if (!this.modelInitialized) {
207-
elizaLogger.info("Model not initialized, starting initialization...");
214+
elizaLogger.info(
215+
"Model not initialized, starting initialization..."
216+
);
208217
await this.initializeModel();
209218
} else {
210219
elizaLogger.info("Model already initialized");
@@ -222,9 +231,13 @@ export class LlamaService extends Service {
222231
);
223232

224233
if (hasCUDA) {
225-
elizaLogger.info("LlamaService: CUDA detected, using GPU acceleration");
234+
elizaLogger.info(
235+
"LlamaService: CUDA detected, using GPU acceleration"
236+
);
226237
} else {
227-
elizaLogger.warn("LlamaService: No CUDA detected - local response will be slow");
238+
elizaLogger.warn(
239+
"LlamaService: No CUDA detected - local response will be slow"
240+
);
228241
}
229242

230243
elizaLogger.info("Initializing Llama instance...");
@@ -252,14 +265,24 @@ export class LlamaService extends Service {
252265
elizaLogger.success("Model initialization complete");
253266
this.processQueue();
254267
} catch (error) {
255-
elizaLogger.error("Model initialization failed. Deleting model and retrying:", error);
268+
elizaLogger.error(
269+
"Model initialization failed. Deleting model and retrying:",
270+
error
271+
);
256272
try {
257-
elizaLogger.info("Attempting to delete and re-download model...");
258-
await this.deleteModel();
259-
await this.initializeModel();
273+
elizaLogger.info(
274+
"Attempting to delete and re-download model..."
275+
);
276+
await this.deleteModel();
277+
await this.initializeModel();
260278
} catch (retryError) {
261-
elizaLogger.error("Model re-initialization failed:", retryError);
262-
throw new Error(`Model initialization failed after retry: ${retryError.message}`);
279+
elizaLogger.error(
280+
"Model re-initialization failed:",
281+
retryError
282+
);
283+
throw new Error(
284+
`Model initialization failed after retry: ${retryError.message}`
285+
);
263286
}
264287
}
265288
}
@@ -275,48 +298,83 @@ export class LlamaService extends Service {
275298
const downloadModel = (url: string) => {
276299
https
277300
.get(url, (response) => {
278-
if (response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
279-
elizaLogger.info(`Following redirect to: ${response.headers.location}`);
301+
if (
302+
response.statusCode >= 300 &&
303+
response.statusCode < 400 &&
304+
response.headers.location
305+
) {
306+
elizaLogger.info(
307+
`Following redirect to: ${response.headers.location}`
308+
);
280309
downloadModel(response.headers.location);
281-
return;
282-
}
310+
return;
311+
}
283312

284313
if (response.statusCode !== 200) {
285-
reject(new Error(`Failed to download model: HTTP ${response.statusCode}`));
314+
reject(
315+
new Error(
316+
`Failed to download model: HTTP ${response.statusCode}`
317+
)
318+
);
286319
return;
287320
}
288321

289-
totalSize = parseInt(response.headers['content-length'] || '0', 10);
290-
elizaLogger.info(`Downloading model: Hermes-3-Llama-3.1-8B.Q8_0.gguf`);
291-
elizaLogger.info(`Download location: ${this.modelPath}`);
292-
elizaLogger.info(`Total size: ${(totalSize / 1024 / 1024).toFixed(2)} MB`);
322+
totalSize = parseInt(
323+
response.headers["content-length"] || "0",
324+
10
325+
);
326+
elizaLogger.info(
327+
`Downloading model: Hermes-3-Llama-3.1-8B.Q8_0.gguf`
328+
);
329+
elizaLogger.info(
330+
`Download location: ${this.modelPath}`
331+
);
332+
elizaLogger.info(
333+
`Total size: ${(totalSize / 1024 / 1024).toFixed(2)} MB`
334+
);
293335

294336
response.pipe(file);
295337

296-
let progressString = '';
338+
let progressString = "";
297339
response.on("data", (chunk) => {
298340
downloadedSize += chunk.length;
299-
const progress = totalSize > 0 ? ((downloadedSize / totalSize) * 100).toFixed(1) : '0.0';
300-
const dots = '.'.repeat(Math.floor(Number(progress) / 5));
301-
progressString = `Downloading model: [${dots.padEnd(20, ' ')}] ${progress}%`;
341+
const progress =
342+
totalSize > 0
343+
? (
344+
(downloadedSize / totalSize) *
345+
100
346+
).toFixed(1)
347+
: "0.0";
348+
const dots = ".".repeat(
349+
Math.floor(Number(progress) / 5)
350+
);
351+
progressString = `Downloading model: [${dots.padEnd(20, " ")}] ${progress}%`;
302352
elizaLogger.progress(progressString);
303353
});
304354

305355
file.on("finish", () => {
306356
file.close();
307-
elizaLogger.progress(''); // Clear the progress line
357+
elizaLogger.progress(""); // Clear the progress line
308358
elizaLogger.success("Model download complete");
309359
resolve();
310360
});
311361

312362
response.on("error", (error) => {
313363
fs.unlink(this.modelPath, () => {});
314-
reject(new Error(`Model download failed: ${error.message}`));
364+
reject(
365+
new Error(
366+
`Model download failed: ${error.message}`
367+
)
368+
);
315369
});
316370
})
317371
.on("error", (error) => {
318372
fs.unlink(this.modelPath, () => {});
319-
reject(new Error(`Model download request failed: ${error.message}`));
373+
reject(
374+
new Error(
375+
`Model download request failed: ${error.message}`
376+
)
377+
);
320378
});
321379
};
322380

@@ -465,12 +523,15 @@ export class LlamaService extends Service {
465523
): Promise<any | string> {
466524
const ollamaModel = process.env.OLLAMA_MODEL;
467525
if (ollamaModel) {
468-
const ollamaUrl = process.env.OLLAMA_SERVER_URL || 'http://localhost:11434';
469-
elizaLogger.info(`Using Ollama API at ${ollamaUrl} with model ${ollamaModel}`);
470-
526+
const ollamaUrl =
527+
process.env.OLLAMA_SERVER_URL || "http://localhost:11434";
528+
elizaLogger.info(
529+
`Using Ollama API at ${ollamaUrl} with model ${ollamaModel}`
530+
);
531+
471532
const response = await fetch(`${ollamaUrl}/api/generate`, {
472-
method: 'POST',
473-
headers: { 'Content-Type': 'application/json' },
533+
method: "POST",
534+
headers: { "Content-Type": "application/json" },
474535
body: JSON.stringify({
475536
model: ollamaModel,
476537
prompt: context,
@@ -481,12 +542,14 @@ export class LlamaService extends Service {
481542
frequency_penalty,
482543
presence_penalty,
483544
num_predict: max_tokens,
484-
}
545+
},
485546
}),
486547
});
487548

488549
if (!response.ok) {
489-
throw new Error(`Ollama request failed: ${response.statusText}`);
550+
throw new Error(
551+
`Ollama request failed: ${response.statusText}`
552+
);
490553
}
491554

492555
const result = await response.json();
@@ -576,21 +639,27 @@ export class LlamaService extends Service {
576639
async getEmbeddingResponse(input: string): Promise<number[] | undefined> {
577640
const ollamaModel = process.env.OLLAMA_MODEL;
578641
if (ollamaModel) {
579-
const ollamaUrl = process.env.OLLAMA_SERVER_URL || 'http://localhost:11434';
580-
const embeddingModel = process.env.OLLAMA_EMBEDDING_MODEL || 'mxbai-embed-large';
581-
elizaLogger.info(`Using Ollama API for embeddings with model ${embeddingModel}`);
582-
642+
const ollamaUrl =
643+
process.env.OLLAMA_SERVER_URL || "http://localhost:11434";
644+
const embeddingModel =
645+
process.env.OLLAMA_EMBEDDING_MODEL || "mxbai-embed-large";
646+
elizaLogger.info(
647+
`Using Ollama API for embeddings with model ${embeddingModel}`
648+
);
649+
583650
const response = await fetch(`${ollamaUrl}/api/embeddings`, {
584-
method: 'POST',
585-
headers: { 'Content-Type': 'application/json' },
651+
method: "POST",
652+
headers: { "Content-Type": "application/json" },
586653
body: JSON.stringify({
587654
model: embeddingModel,
588655
prompt: input,
589656
}),
590657
});
591658

592659
if (!response.ok) {
593-
throw new Error(`Ollama embeddings request failed: ${response.statusText}`);
660+
throw new Error(
661+
`Ollama embeddings request failed: ${response.statusText}`
662+
);
594663
}
595664

596665
const result = await response.json();
@@ -609,12 +678,15 @@ export class LlamaService extends Service {
609678

610679
private async ollamaCompletion(prompt: string): Promise<string> {
611680
const ollamaModel = process.env.OLLAMA_MODEL;
612-
const ollamaUrl = process.env.OLLAMA_SERVER_URL || 'http://localhost:11434';
613-
elizaLogger.info(`Using Ollama API at ${ollamaUrl} with model ${ollamaModel}`);
614-
681+
const ollamaUrl =
682+
process.env.OLLAMA_SERVER_URL || "http://localhost:11434";
683+
elizaLogger.info(
684+
`Using Ollama API at ${ollamaUrl} with model ${ollamaModel}`
685+
);
686+
615687
const response = await fetch(`${ollamaUrl}/api/generate`, {
616-
method: 'POST',
617-
headers: { 'Content-Type': 'application/json' },
688+
method: "POST",
689+
headers: { "Content-Type": "application/json" },
618690
body: JSON.stringify({
619691
model: ollamaModel,
620692
prompt: prompt,
@@ -625,7 +697,7 @@ export class LlamaService extends Service {
625697
frequency_penalty: 0.5,
626698
presence_penalty: 0.5,
627699
num_predict: 256,
628-
}
700+
},
629701
}),
630702
});
631703

@@ -639,21 +711,27 @@ export class LlamaService extends Service {
639711

640712
private async ollamaEmbedding(text: string): Promise<number[]> {
641713
const ollamaModel = process.env.OLLAMA_MODEL;
642-
const ollamaUrl = process.env.OLLAMA_SERVER_URL || 'http://localhost:11434';
643-
const embeddingModel = process.env.OLLAMA_EMBEDDING_MODEL || 'mxbai-embed-large';
644-
elizaLogger.info(`Using Ollama API for embeddings with model ${embeddingModel}`);
645-
714+
const ollamaUrl =
715+
process.env.OLLAMA_SERVER_URL || "http://localhost:11434";
716+
const embeddingModel =
717+
process.env.OLLAMA_EMBEDDING_MODEL || "mxbai-embed-large";
718+
elizaLogger.info(
719+
`Using Ollama API for embeddings with model ${embeddingModel}`
720+
);
721+
646722
const response = await fetch(`${ollamaUrl}/api/embeddings`, {
647-
method: 'POST',
648-
headers: { 'Content-Type': 'application/json' },
723+
method: "POST",
724+
headers: { "Content-Type": "application/json" },
649725
body: JSON.stringify({
650726
model: embeddingModel,
651727
prompt: text,
652728
}),
653729
});
654730

655731
if (!response.ok) {
656-
throw new Error(`Ollama embeddings request failed: ${response.statusText}`);
732+
throw new Error(
733+
`Ollama embeddings request failed: ${response.statusText}`
734+
);
657735
}
658736

659737
const result = await response.json();

0 commit comments

Comments
 (0)