diff --git a/package.json b/package.json
index 372d54db50..0d394aa59b 100644
--- a/package.json
+++ b/package.json
@@ -47,7 +47,8 @@
"dependencies": {
"ollama-ai-provider": "^0.16.1",
"optional": "^0.1.4",
- "sharp": "^0.33.5"
+ "sharp": "^0.33.5",
+ "tslog": "^4.9.3"
},
"packageManager": "pnpm@9.12.3+sha512.cce0f9de9c5a7c95bef944169cc5dfe8741abfb145078c0d508b868056848a87c81e626246cb60967cbd7fd29a6c062ef73ff840d96b3c86c40ac92cf4a813ee"
}
diff --git a/packages/core/src/defaultCharacter.ts b/packages/core/src/defaultCharacter.ts
index 2ca1bf0179..1a1b4e7583 100644
--- a/packages/core/src/defaultCharacter.ts
+++ b/packages/core/src/defaultCharacter.ts
@@ -5,7 +5,7 @@ export const defaultCharacter: Character = {
username: "eliza",
plugins: [],
clients: [],
- modelProvider: ModelProviderName.OPENAI,
+ modelProvider: ModelProviderName.OLLAMA,
settings: {
secrets: {},
voice: {
diff --git a/packages/core/src/embedding.ts b/packages/core/src/embedding.ts
index 0bb91f136b..83b287fa78 100644
--- a/packages/core/src/embedding.ts
+++ b/packages/core/src/embedding.ts
@@ -86,8 +86,10 @@ export async function embed(runtime: IAgentRuntime, input: string) {
// 3. Fallback to OpenAI embedding model
const embeddingModel = settings.USE_OPENAI_EMBEDDING
? "text-embedding-3-small"
- : modelProvider.model?.[ModelClass.EMBEDDING] ||
- models[ModelProviderName.OPENAI].model[ModelClass.EMBEDDING];
+ : runtime.character.modelProvider === ModelProviderName.OLLAMA
+ ? settings.OLLAMA_EMBEDDING_MODEL || "mxbai-embed-large"
+ : modelProvider.model?.[ModelClass.EMBEDDING] ||
+ models[ModelProviderName.OPENAI].model[ModelClass.EMBEDDING];
if (!embeddingModel) {
throw new Error("No embedding model configured");
diff --git a/packages/core/src/generation.ts b/packages/core/src/generation.ts
index f20004ea9b..d58a0fdbde 100644
--- a/packages/core/src/generation.ts
+++ b/packages/core/src/generation.ts
@@ -62,7 +62,12 @@ export async function generateText({
return "";
}
- elizaLogger.log("Genarating text...");
+ elizaLogger.log("Generating text...");
+
+ elizaLogger.info("Generating text with options:", {
+ modelProvider: runtime.modelProvider,
+ model: modelClass,
+ });
const provider = runtime.modelProvider;
const endpoint =
@@ -84,6 +89,8 @@ export async function generateText({
model = runtime.getSetting("LLAMACLOUD_MODEL_SMALL");
}
+ elizaLogger.info("Selected model:", model);
+
const temperature = models[provider].settings.temperature;
const frequency_penalty = models[provider].settings.frequency_penalty;
const presence_penalty = models[provider].settings.presence_penalty;
@@ -709,7 +716,7 @@ export async function generateMessageResponse({
let retryLength = 1000; // exponential backoff
while (true) {
try {
- elizaLogger.log("Genarating message response..");
+ elizaLogger.log("Generating message response..");
const response = await generateText({
runtime,
diff --git a/packages/core/src/logger.ts b/packages/core/src/logger.ts
index f8172d0b6c..ae9b3a1985 100644
--- a/packages/core/src/logger.ts
+++ b/packages/core/src/logger.ts
@@ -1,4 +1,11 @@
-class ElizaLogger {
+import settings from "./settings.ts";
+import { Logger, ILogObjMeta, ILogObj } from "tslog";
+
+interface IElizaLogger extends Logger<IElizaLogger> {
+ progress(message: string): void;
+}
+
+class ElizaLogger implements IElizaLogger {
constructor() {
// Check if we're in Node.js environment
this.isNode =
@@ -7,7 +14,7 @@ class ElizaLogger {
process.versions.node != null;
// Set verbose based on environment
- this.verbose = this.isNode ? process.env.verbose === "true" : false;
+ this.verbose = this.isNode ? settings.VERBOSE === "true" : false;
}
private isNode: boolean;
@@ -173,6 +180,7 @@ class ElizaLogger {
}
}
+ // @ts-ignore - custom implementation
log(...strings) {
this.#logWithStyle(strings, {
fg: "white",
@@ -182,6 +190,7 @@ class ElizaLogger {
});
}
+ // @ts-ignore - custom implementation
warn(...strings) {
this.#logWithStyle(strings, {
fg: "yellow",
@@ -191,6 +200,7 @@ class ElizaLogger {
});
}
+ // @ts-ignore - custom implementation
error(...strings) {
this.#logWithStyle(strings, {
fg: "red",
@@ -200,6 +210,7 @@ class ElizaLogger {
});
}
+ // @ts-ignore - custom implementation
info(...strings) {
this.#logWithStyle(strings, {
fg: "blue",
@@ -209,15 +220,7 @@ class ElizaLogger {
});
}
- success(...strings) {
- this.#logWithStyle(strings, {
- fg: "green",
- bg: "",
- icon: "\u2713",
- groupTitle: ` ${this.successesTitle}`,
- });
- }
-
+ // @ts-ignore - custom implementation
debug(...strings) {
if (!this.verbose) return;
this.#logWithStyle(strings, {
@@ -228,6 +231,15 @@ class ElizaLogger {
});
}
+ success(...strings) {
+ this.#logWithStyle(strings, {
+ fg: "green",
+ bg: "",
+ icon: "\u2713",
+ groupTitle: ` ${this.successesTitle}`,
+ });
+ }
+
assert(...strings) {
this.#logWithStyle(strings, {
fg: "cyan",
@@ -236,6 +248,17 @@ class ElizaLogger {
groupTitle: ` ${this.assertsTitle}`,
});
}
+
+ progress(message: string) {
+ if (this.isNode) {
+ // Clear the current line and move cursor to beginning
+ process.stdout.clearLine(0);
+ process.stdout.cursorTo(0);
+ process.stdout.write(message);
+ } else {
+ console.log(message);
+ }
+ }
}
export const elizaLogger = new ElizaLogger();
diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts
index f973b321d6..825ca3e2fc 100644
--- a/packages/core/src/runtime.ts
+++ b/packages/core/src/runtime.ts
@@ -176,7 +176,9 @@ export class AgentRuntime implements IAgentRuntime {
return;
}
+ // Add the service to the services map
this.services.set(serviceType, service);
+ elizaLogger.success(`Service ${serviceType} registered successfully`);
}
/**
@@ -217,6 +219,12 @@ export class AgentRuntime implements IAgentRuntime {
cacheManager: ICacheManager;
logging?: boolean;
}) {
+ elizaLogger.info("Initializing AgentRuntime with options:", {
+ character: opts.character?.name,
+ modelProvider: opts.modelProvider,
+ characterModelProvider: opts.character?.modelProvider,
+ });
+
this.#conversationLength =
opts.conversationLength ?? this.#conversationLength;
this.databaseAdapter = opts.databaseAdapter;
@@ -280,10 +288,32 @@ export class AgentRuntime implements IAgentRuntime {
});
this.serverUrl = opts.serverUrl ?? this.serverUrl;
+
+ elizaLogger.info("Setting model provider...");
+ elizaLogger.info(
+ "- Character model provider:",
+ this.character.modelProvider
+ );
+ elizaLogger.info("- Opts model provider:", opts.modelProvider);
+ elizaLogger.info("- Current model provider:", this.modelProvider);
+
this.modelProvider =
this.character.modelProvider ??
opts.modelProvider ??
this.modelProvider;
+
+ elizaLogger.info("Selected model provider:", this.modelProvider);
+
+ // Validate model provider
+ if (!Object.values(ModelProviderName).includes(this.modelProvider)) {
+ elizaLogger.error("Invalid model provider:", this.modelProvider);
+ elizaLogger.error(
+ "Available providers:",
+ Object.values(ModelProviderName)
+ );
+ throw new Error(`Invalid model provider: ${this.modelProvider}`);
+ }
+
if (!this.serverUrl) {
elizaLogger.warn("No serverUrl provided, defaulting to localhost");
}
diff --git a/packages/plugin-node/src/services/image.ts b/packages/plugin-node/src/services/image.ts
index 3f60566f66..d840e7f1f4 100644
--- a/packages/plugin-node/src/services/image.ts
+++ b/packages/plugin-node/src/services/image.ts
@@ -63,7 +63,7 @@ export class ImageDescriptionService
env.backends.onnx.wasm.proxy = false;
env.backends.onnx.wasm.numThreads = 1;
- elizaLogger.log("Downloading Florence model...");
+ elizaLogger.info("Downloading Florence model...");
this.model = await Florence2ForConditionalGeneration.from_pretrained(
this.modelId,
@@ -71,8 +71,15 @@ export class ImageDescriptionService
device: "gpu",
progress_callback: (progress) => {
if (progress.status === "downloading") {
- elizaLogger.log(
- `Model download progress: ${JSON.stringify(progress)}`
+ const percent = (
+ (progress.loaded / progress.total) *
+ 100
+ ).toFixed(1);
+ const dots = ".".repeat(
+ Math.floor(Number(percent) / 5)
+ );
+ elizaLogger.info(
+ `Downloading Florence model: [${dots.padEnd(20, " ")}] ${percent}%`
);
}
},
@@ -81,10 +88,14 @@ export class ImageDescriptionService
elizaLogger.success("Florence model downloaded successfully");
+ elizaLogger.info("Downloading processor...");
this.processor = (await AutoProcessor.from_pretrained(
this.modelId
)) as Florence2Processor;
+
+ elizaLogger.info("Downloading tokenizer...");
this.tokenizer = await AutoTokenizer.from_pretrained(this.modelId);
+ elizaLogger.success("Image service initialization complete");
}
async describeImage(
diff --git a/packages/plugin-node/src/services/llama.ts b/packages/plugin-node/src/services/llama.ts
index 7a747136c0..0a08b3754f 100644
--- a/packages/plugin-node/src/services/llama.ts
+++ b/packages/plugin-node/src/services/llama.ts
@@ -1,4 +1,9 @@
-import { elizaLogger, IAgentRuntime, ServiceType } from "@ai16z/eliza";
+import {
+ elizaLogger,
+ IAgentRuntime,
+ ServiceType,
+ ModelProviderName,
+} from "@ai16z/eliza";
import { Service } from "@ai16z/eliza";
import fs from "fs";
import https from "https";
@@ -164,6 +169,7 @@ export class LlamaService extends Service {
private ctx: LlamaContext | undefined;
private sequence: LlamaContextSequence | undefined;
private modelUrl: string;
+ private ollamaModel: string | undefined;
private messageQueue: QueuedMessage[] = [];
private isProcessing: boolean = false;
@@ -179,18 +185,46 @@ export class LlamaService extends Service {
"https://huggingface.co/NousResearch/Hermes-3-Llama-3.1-8B-GGUF/resolve/main/Hermes-3-Llama-3.1-8B.Q8_0.gguf?download=true";
const modelName = "model.gguf";
this.modelPath = path.join(__dirname, modelName);
+ this.ollamaModel = process.env.OLLAMA_MODEL;
}
- async initialize(_runtime: IAgentRuntime): Promise<void> {}
+ async initialize(runtime: IAgentRuntime): Promise<void> {
+ elizaLogger.info("Initializing LlamaService...");
+ try {
+ // Check if we should use Ollama
+ if (runtime.modelProvider === ModelProviderName.OLLAMA) {
+ elizaLogger.info("Using Ollama provider");
+ this.modelInitialized = true;
+ return;
+ }
+
+ elizaLogger.info("Using local GGUF model");
+ elizaLogger.info("Ensuring model is initialized...");
+ await this.ensureInitialized();
+ elizaLogger.success("LlamaService initialized successfully");
+ } catch (error) {
+ elizaLogger.error("Failed to initialize LlamaService:", error);
+ // Re-throw with more context
+ throw new Error(
+ `LlamaService initialization failed: ${error.message}`
+ );
+ }
+ }
private async ensureInitialized() {
if (!this.modelInitialized) {
+ elizaLogger.info(
+ "Model not initialized, starting initialization..."
+ );
await this.initializeModel();
+ } else {
+ elizaLogger.info("Model already initialized");
}
}
async initializeModel() {
try {
+ elizaLogger.info("Checking model file...");
await this.checkModel();
const systemInfo = await si.graphics();
@@ -199,92 +233,150 @@ export class LlamaService extends Service {
);
if (hasCUDA) {
- console.log("**** LlamaService: CUDA detected");
+ elizaLogger.info(
+ "LlamaService: CUDA detected, using GPU acceleration"
+ );
} else {
- console.warn(
- "**** LlamaService: No CUDA detected - local response will be slow"
+ elizaLogger.warn(
+ "LlamaService: No CUDA detected - local response will be slow"
);
}
+ elizaLogger.info("Initializing Llama instance...");
this.llama = await getLlama({
- gpu: "cuda",
+ gpu: hasCUDA ? "cuda" : undefined,
});
+
+ elizaLogger.info("Creating JSON schema grammar...");
const grammar = new LlamaJsonSchemaGrammar(
this.llama,
jsonSchemaGrammar as GbnfJsonSchema
);
this.grammar = grammar;
+ elizaLogger.info("Loading model...");
this.model = await this.llama.loadModel({
modelPath: this.modelPath,
});
+ elizaLogger.info("Creating context and sequence...");
this.ctx = await this.model.createContext({ contextSize: 8192 });
this.sequence = this.ctx.getSequence();
this.modelInitialized = true;
+ elizaLogger.success("Model initialization complete");
this.processQueue();
} catch (error) {
- console.error(
- "Model initialization failed. Deleting model and retrying...",
+ elizaLogger.error(
+ "Model initialization failed. Deleting model and retrying:",
error
);
- await this.deleteModel();
- await this.initializeModel();
+ try {
+ elizaLogger.info(
+ "Attempting to delete and re-download model..."
+ );
+ await this.deleteModel();
+ await this.initializeModel();
+ } catch (retryError) {
+ elizaLogger.error(
+ "Model re-initialization failed:",
+ retryError
+ );
+ throw new Error(
+ `Model initialization failed after retry: ${retryError.message}`
+ );
+ }
}
}
async checkModel() {
if (!fs.existsSync(this.modelPath)) {
+ elizaLogger.info("Model file not found, starting download...");
await new Promise<void>((resolve, reject) => {
const file = fs.createWriteStream(this.modelPath);
let downloadedSize = 0;
+ let totalSize = 0;
const downloadModel = (url: string) => {
https
.get(url, (response) => {
- const isRedirect =
+ if (
response.statusCode >= 300 &&
- response.statusCode < 400;
- if (isRedirect) {
- const redirectUrl = response.headers.location;
- if (redirectUrl) {
- downloadModel(redirectUrl);
- return;
- } else {
- reject(new Error("Redirect URL not found"));
- return;
- }
+ response.statusCode < 400 &&
+ response.headers.location
+ ) {
+ elizaLogger.info(
+ `Following redirect to: ${response.headers.location}`
+ );
+ downloadModel(response.headers.location);
+ return;
+ }
+
+ if (response.statusCode !== 200) {
+ reject(
+ new Error(
+ `Failed to download model: HTTP ${response.statusCode}`
+ )
+ );
+ return;
}
- const totalSize = parseInt(
- response.headers["content-length"] ?? "0",
+ totalSize = parseInt(
+ response.headers["content-length"] || "0",
10
);
+ elizaLogger.info(
+ `Downloading model: Hermes-3-Llama-3.1-8B.Q8_0.gguf`
+ );
+ elizaLogger.info(
+ `Download location: ${this.modelPath}`
+ );
+ elizaLogger.info(
+ `Total size: ${(totalSize / 1024 / 1024).toFixed(2)} MB`
+ );
+
+ response.pipe(file);
+ let progressString = "";
response.on("data", (chunk) => {
downloadedSize += chunk.length;
- file.write(chunk);
-
- // Log progress
- const progress = (
- (downloadedSize / totalSize) *
- 100
- ).toFixed(2);
- process.stdout.write(
- `Downloaded ${progress}%\r`
+ const progress =
+ totalSize > 0
+ ? (
+ (downloadedSize / totalSize) *
+ 100
+ ).toFixed(1)
+ : "0.0";
+ const dots = ".".repeat(
+ Math.floor(Number(progress) / 5)
);
+ progressString = `Downloading model: [${dots.padEnd(20, " ")}] ${progress}%`;
+ elizaLogger.progress(progressString);
});
- response.on("end", () => {
- file.end();
+ file.on("finish", () => {
+ file.close();
+ elizaLogger.progress(""); // Clear the progress line
+ elizaLogger.success("Model download complete");
resolve();
});
+
+ response.on("error", (error) => {
+ fs.unlink(this.modelPath, () => {});
+ reject(
+ new Error(
+ `Model download failed: ${error.message}`
+ )
+ );
+ });
})
- .on("error", (err) => {
- fs.unlink(this.modelPath, () => {}); // Delete the file async
- console.error("Download failed:", err.message);
- reject(err);
+ .on("error", (error) => {
+ fs.unlink(this.modelPath, () => {});
+ reject(
+ new Error(
+ `Model download request failed: ${error.message}`
+ )
+ );
});
};
@@ -392,6 +484,36 @@ export class LlamaService extends Service {
this.isProcessing = false;
}
+ async completion(prompt: string, runtime: IAgentRuntime): Promise<string> {
+ try {
+ await this.initialize(runtime);
+
+ if (runtime.modelProvider === ModelProviderName.OLLAMA) {
+ return await this.ollamaCompletion(prompt);
+ }
+
+ return await this.localCompletion(prompt);
+ } catch (error) {
+ elizaLogger.error("Error in completion:", error);
+ throw error;
+ }
+ }
+
+ async embedding(text: string, runtime: IAgentRuntime): Promise<number[]> {
+ try {
+ await this.initialize(runtime);
+
+ if (runtime.modelProvider === ModelProviderName.OLLAMA) {
+ return await this.ollamaEmbedding(text);
+ }
+
+ return await this.localEmbedding(text);
+ } catch (error) {
+ elizaLogger.error("Error in embedding:", error);
+ throw error;
+ }
+ }
+
private async getCompletionResponse(
context: string,
temperature: number,
@@ -401,6 +523,42 @@ export class LlamaService extends Service {
max_tokens: number,
useGrammar: boolean
): Promise<any | string> {
+ const ollamaModel = process.env.OLLAMA_MODEL;
+ if (ollamaModel) {
+ const ollamaUrl =
+ process.env.OLLAMA_SERVER_URL || "http://localhost:11434";
+ elizaLogger.info(
+ `Using Ollama API at ${ollamaUrl} with model ${ollamaModel}`
+ );
+
+ const response = await fetch(`${ollamaUrl}/api/generate`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ model: ollamaModel,
+ prompt: context,
+ stream: false,
+ options: {
+ temperature,
+ stop,
+ frequency_penalty,
+ presence_penalty,
+ num_predict: max_tokens,
+ },
+ }),
+ });
+
+ if (!response.ok) {
+ throw new Error(
+ `Ollama request failed: ${response.statusText}`
+ );
+ }
+
+ const result = await response.json();
+ return useGrammar ? { content: result.response } : result.response;
+ }
+
+ // Use local GGUF model
if (!this.sequence) {
throw new Error("Model not initialized.");
}
@@ -429,7 +587,7 @@ export class LlamaService extends Service {
})) {
const current = this.model.detokenize([...responseTokens, token]);
if ([...stop].some((s) => current.includes(s))) {
- console.log("Stop sequence found");
+ elizaLogger.info("Stop sequence found");
break;
}
@@ -437,12 +595,12 @@ export class LlamaService extends Service {
process.stdout.write(this.model!.detokenize([token]));
if (useGrammar) {
if (current.replaceAll("\n", "").includes("}```")) {
- console.log("JSON block found");
+ elizaLogger.info("JSON block found");
break;
}
}
if (responseTokens.length > max_tokens) {
- console.log("Max tokens reached");
+ elizaLogger.info("Max tokens reached");
break;
}
}
@@ -472,7 +630,7 @@ export class LlamaService extends Service {
await this.sequence.clearHistory();
return parsedResponse;
} catch (error) {
- console.error("Error parsing JSON:", error);
+ elizaLogger.error("Error parsing JSON:", error);
}
} else {
await this.sequence.clearHistory();
@@ -481,13 +639,186 @@ export class LlamaService extends Service {
}
async getEmbeddingResponse(input: string): Promise<number[] | undefined> {
- await this.ensureInitialized();
- if (!this.model) {
- throw new Error("Model not initialized. Call initialize() first.");
+ const ollamaModel = process.env.OLLAMA_MODEL;
+ if (ollamaModel) {
+ const ollamaUrl =
+ process.env.OLLAMA_SERVER_URL || "http://localhost:11434";
+ const embeddingModel =
+ process.env.OLLAMA_EMBEDDING_MODEL || "mxbai-embed-large";
+ elizaLogger.info(
+ `Using Ollama API for embeddings with model ${embeddingModel} (base: ${ollamaModel})`
+ );
+
+ const response = await fetch(`${ollamaUrl}/api/embeddings`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ model: embeddingModel,
+ prompt: input,
+ }),
+ });
+
+ if (!response.ok) {
+ throw new Error(
+ `Ollama embeddings request failed: ${response.statusText}`
+ );
+ }
+
+ const result = await response.json();
+ return result.embedding;
+ }
+
+ // Use local GGUF model
+ if (!this.sequence) {
+ throw new Error("Sequence not initialized");
+ }
+
+ const ollamaUrl =
+ process.env.OLLAMA_SERVER_URL || "http://localhost:11434";
+ const embeddingModel =
+ process.env.OLLAMA_EMBEDDING_MODEL || "mxbai-embed-large";
+ elizaLogger.info(
+ `Using Ollama API for embeddings with model ${embeddingModel} (base: ${this.ollamaModel})`
+ );
+
+ const response = await fetch(`${ollamaUrl}/api/embeddings`, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ input: input,
+ model: embeddingModel,
+ }),
+ });
+
+ if (!response.ok) {
+ throw new Error(`Failed to get embedding: ${response.statusText}`);
+ }
+
+ const embedding = await response.json();
+ return embedding.vector;
+ }
+
+ private async ollamaCompletion(prompt: string): Promise<string> {
+ const ollamaModel = process.env.OLLAMA_MODEL;
+ const ollamaUrl =
+ process.env.OLLAMA_SERVER_URL || "http://localhost:11434";
+ elizaLogger.info(
+ `Using Ollama API at ${ollamaUrl} with model ${ollamaModel}`
+ );
+
+ const response = await fetch(`${ollamaUrl}/api/generate`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ model: ollamaModel,
+ prompt: prompt,
+ stream: false,
+ options: {
+ temperature: 0.7,
+ stop: ["\n"],
+ frequency_penalty: 0.5,
+ presence_penalty: 0.5,
+ num_predict: 256,
+ },
+ }),
+ });
+
+ if (!response.ok) {
+ throw new Error(`Ollama request failed: ${response.statusText}`);
+ }
+
+ const result = await response.json();
+ return result.response;
+ }
+
+ private async ollamaEmbedding(text: string): Promise<number[]> {
+ const ollamaModel = process.env.OLLAMA_MODEL;
+ const ollamaUrl =
+ process.env.OLLAMA_SERVER_URL || "http://localhost:11434";
+ const embeddingModel =
+ process.env.OLLAMA_EMBEDDING_MODEL || "mxbai-embed-large";
+ elizaLogger.info(
+ `Using Ollama API for embeddings with model ${embeddingModel} (base: ${ollamaModel})`
+ );
+
+ const response = await fetch(`${ollamaUrl}/api/embeddings`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ model: embeddingModel,
+ prompt: text,
+ }),
+ });
+
+ if (!response.ok) {
+ throw new Error(
+ `Ollama embeddings request failed: ${response.statusText}`
+ );
+ }
+
+ const result = await response.json();
+ return result.embedding;
+ }
+
+ private async localCompletion(prompt: string): Promise<string> {
+ if (!this.sequence) {
+ throw new Error("Sequence not initialized");
+ }
+
+ const tokens = this.model!.tokenize(prompt);
+
+ // tokenize the words to punish
+ const wordsToPunishTokens = wordsToPunish
+ .map((word) => this.model!.tokenize(word))
+ .flat();
+
+ const repeatPenalty: LlamaContextSequenceRepeatPenalty = {
+ punishTokens: () => wordsToPunishTokens,
+ penalty: 1.2,
+ frequencyPenalty: 0.5,
+ presencePenalty: 0.5,
+ };
+
+ const responseTokens: Token[] = [];
+
+ for await (const token of this.sequence.evaluate(tokens, {
+ temperature: 0.7,
+ repeatPenalty: repeatPenalty,
+ yieldEogToken: false,
+ })) {
+ const current = this.model.detokenize([...responseTokens, token]);
+ if (current.includes("\n")) {
+ elizaLogger.info("Stop sequence found");
+ break;
+ }
+
+ responseTokens.push(token);
+ process.stdout.write(this.model!.detokenize([token]));
+ if (responseTokens.length > 256) {
+ elizaLogger.info("Max tokens reached");
+ break;
+ }
+ }
+
+ const response = this.model!.detokenize(responseTokens);
+
+ if (!response) {
+ throw new Error("Response is undefined");
+ }
+
+ await this.sequence.clearHistory();
+ return response;
+ }
+
+ private async localEmbedding(text: string): Promise<number[]> {
+ if (!this.sequence) {
+ throw new Error("Sequence not initialized");
}
const embeddingContext = await this.model.createEmbeddingContext();
- const embedding = await embeddingContext.getEmbeddingFor(input);
+ const embedding = await embeddingContext.getEmbeddingFor(text);
return embedding?.vector ? [...embedding.vector] : undefined;
}
}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index e7cddacf7d..5e7297788e 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -20,6 +20,9 @@ importers:
sharp:
specifier: ^0.33.5
version: 0.33.5
+ tslog:
+ specifier: ^4.9.3
+ version: 4.9.3
devDependencies:
'@commitlint/cli':
specifier: ^18.4.4
@@ -13980,6 +13983,10 @@ packages:
tslib@2.8.0:
resolution: {integrity: sha512-jWVzBLplnCmoaTr13V9dYbiQ99wvZRd0vNWaDRg+aVYRcjDF3nDksxFDE/+fkXnKhpnUUkmx5pK/v8mCtLVqZA==}
+ tslog@4.9.3:
+ resolution: {integrity: sha512-oDWuGVONxhVEBtschLf2cs/Jy8i7h1T+CpdkTNWQgdAF7DhRo2G8vMCgILKe7ojdEkLhICWgI1LYSSKaJsRgcw==}
+ engines: {node: '>=16'}
+
tsup@8.3.5:
resolution: {integrity: sha512-Tunf6r6m6tnZsG9GYWndg0z8dEV7fD733VBFzFJ5Vcm1FtlXB8xBD/rtrBi2a3YKEV7hHtxiZtW5EAVADoe1pA==}
engines: {node: '>=18'}
@@ -31192,6 +31199,8 @@ snapshots:
tslib@2.8.0: {}
+ tslog@4.9.3: {}
+
tsup@8.3.5(@swc/core@1.9.2(@swc/helpers@0.5.15))(jiti@2.4.0)(postcss@8.4.49)(typescript@5.6.3)(yaml@2.6.1):
dependencies:
bundle-require: 5.0.0(esbuild@0.24.0)