add chat formatters

Author: hazelnutcloud

godot/addons/godot-llama-cpp/chat/chat_formatter.gd

@@ -0,0 +1,39 @@
+class_name ChatFormatter
+
+static func apply(format: String, messages: Array) -> String:
+	match format:
+		"llama3":
+			return format_llama3(messages)
+		"phi3":
+			return format_phi3(messages)
+		_:
+			printerr("Unknown chat format: ", format)
+			return ""
+
+static func format_llama3(messages: Array) -> String:
+	var res = "<|begin_of_text|>"
+	
+	for i in range(messages.size()):
+		match messages[i]:
+			{"text": var text, "sender": var sender}:
+				res += """<|start_header_id|>%s<|end_header_id|>
+
+%s<|eot_id|>
+""" % [sender, text]
+			_:
+				printerr("Invalid message at index ", i)
+
+	res += "<|start_header_id|>assistant<|end_header_id|>\n\n"
+	return res
+
+static func format_phi3(messages: Array) -> String:
+	var res = "<s>"
+	
+	for i in range(messages.size()):
+		match messages[i]:
+			{"text": var text, "sender": var sender}:
+				res +="<|%s|>\n%s<|end|>\n" % [sender, text]
+			_:
+				printerr("Invalid message at index ", i)
+	res += "<|assistant|>\n"
+	return res

godot/examples/simple/message.gd

@@ -4,6 +4,12 @@ extends Node
 @onready var text_container = %Text
 @onready var icon = %Panel
 @export_enum("user", "assistant") var sender: String
+@export var include_in_prompt: bool = true
+var text:
+	get:
+		return text_container.text
+	set(value):
+		text_container.text = value
 
 var completion_id: int = -1
 var pending: bool = false

godot/examples/simple/simple.gd

@@ -9,7 +9,16 @@ func _on_text_edit_submit(input: String) -> void:
 	handle_input(input)
 
 func handle_input(input: String) -> void:
-	var completion_id = llama_context.request_completion(input)
+	var messages = [{ "sender": "system", "text": "You are a helpful assistant" }]
+	messages.append_array(messages_container.get_children().filter(func(msg: Message): return msg.include_in_prompt).map(
+		func(msg: Message) -> Dictionary:
+			return { "text": msg.text, "sender": msg.sender }
+	))
+	messages.append({"text": input, "sender": "user"})
+	var prompt = ChatFormatter.apply("phi3", messages)
+	print("prompt: ", prompt)
+	
+	var completion_id = llama_context.request_completion(prompt)
 
 	var user_message: Message = message.instantiate()
 	messages_container.add_child(user_message)
@@ -22,19 +31,20 @@ func handle_input(input: String) -> void:
 	ai_message.sender = "assistant"
 	ai_message.completion_id = completion_id
 	ai_message.pending = true
+	ai_message.grab_focus()
 
 
 
 func _on_llama_context_completion_generated(chunk: Dictionary) -> void:
 	var completion_id = chunk.id
-	for message: Message in messages_container.get_children():
-		if message.completion_id != completion_id or message.sender != "assistant":
+	for msg: Message in messages_container.get_children():
+		if msg.completion_id != completion_id or msg.sender != "assistant":
 			continue
 		if chunk.has("error"):
-			message.errored = true
+			msg.errored = true
 		elif chunk.has("text"):
-			if message.pending:
-				message.pending = false
-				message.set_text(chunk["text"])
+			if msg.pending:
+				msg.pending = false
+				msg.set_text(chunk["text"])
 			else:
-				message.append_text(chunk["text"])
+				msg.append_text(chunk["text"])

godot/examples/simple/simple.tscn

@@ -1,10 +1,10 @@
 [gd_scene load_steps=6 format=3 uid="uid://c55kb4qvg6geq"]
 
-[ext_resource type="LlamaModel" path="res://models/Phi-3-mini-128k-instruct.Q5_K_M.gguf" id="1_ff70a"]
 [ext_resource type="Texture2D" uid="uid://dplw232htshgc" path="res://addons/godot-llama-cpp/assets/godot-llama-cpp-1024x1024.svg" id="1_gjsev"]
 [ext_resource type="Script" path="res://examples/simple/simple.gd" id="1_sruc3"]
 [ext_resource type="PackedScene" uid="uid://t862t0v8ht2q" path="res://examples/simple/message.tscn" id="2_7iip7"]
 [ext_resource type="Script" path="res://examples/simple/TextEdit.gd" id="2_7usqw"]
+[ext_resource type="LlamaModel" path="res://models/Phi-3-mini-128k-instruct.Q5_K_M.gguf" id="5_qpeda"]
 
 [node name="Node" type="Node"]
 script = ExtResource("1_sruc3")
@@ -34,6 +34,7 @@ layout_mode = 2
 [node name="ScrollContainer" type="ScrollContainer" parent="Panel/MarginContainer/VBoxContainer"]
 layout_mode = 2
 size_flags_vertical = 3
+follow_focus = true
 
 [node name="MessagesContainer" type="VBoxContainer" parent="Panel/MarginContainer/VBoxContainer/ScrollContainer"]
 unique_name_in_owner = true
@@ -44,6 +45,7 @@ theme_override_constants/separation = 30
 
 [node name="RichTextLabel2" parent="Panel/MarginContainer/VBoxContainer/ScrollContainer/MessagesContainer" instance=ExtResource("2_7iip7")]
 layout_mode = 2
+include_in_prompt = false
 
 [node name="Text" parent="Panel/MarginContainer/VBoxContainer/ScrollContainer/MessagesContainer/RichTextLabel2" index="1"]
 text = "How can I help you?"
@@ -66,7 +68,7 @@ icon = ExtResource("1_gjsev")
 expand_icon = true
 
 [node name="LlamaContext" type="LlamaContext" parent="."]
-model = ExtResource("1_ff70a")
+model = ExtResource("5_qpeda")
 unique_name_in_owner = true
 
 [connection signal="submit" from="Panel/MarginContainer/VBoxContainer/HBoxContainer/TextEdit" to="." method="_on_text_edit_submit"]

llama.cpp

@@ -96,7 +96,7 @@ void LlamaContext::__thread_loop() {
 		UtilityFunctions::print(vformat("%s: Running completion for prompt id: %d", __func__, req.id));
 
 		std::vector<llama_token> request_tokens;
-		request_tokens = ::llama_tokenize(ctx, req.prompt.utf8().get_data(), true);
+		request_tokens = ::llama_tokenize(ctx, req.prompt.utf8().get_data(), true, true);
 
 		size_t shared_prefix_idx = 0;
 		auto diff = std::mismatch(context_tokens.begin(), context_tokens.end(), request_tokens.begin(), request_tokens.end());
@@ -166,7 +166,7 @@ void LlamaContext::__thread_loop() {
 				return;
 			}
 			llama_token new_token_id = llama_sampling_sample(sampling_ctx, ctx, NULL, batch.n_tokens - 1);
-			llama_sampling_accept(sampling_ctx, ctx, new_token_id, true);
+			llama_sampling_accept(sampling_ctx, ctx, new_token_id, false);
 
 			Dictionary response;
 			response["id"] = req.id;