Update inference code

godot/.gitattributes

@@ -0,0 +1,2 @@
+# Normalize EOL for all files that Git considers text files.
+* text=auto eol=lf

godot/.gitignore

@@ -0,0 +1,2 @@
+# Godot 4+ specific ignores
+.godot/

godot/addons/godot-llama-cpp/chat/chat_formatter.gd

@@ -0,0 +1,56 @@
+class_name ChatFormatter
+
+static func apply(format: String, messages: Array) -> String:
+	match format:
+		"llama3":
+			return format_llama3(messages)
+		"phi3":
+			return format_phi3(messages)
+		"mistral":
+			return format_mistral(messages)
+		_:
+			printerr("Unknown chat format: ", format)
+			return ""
+
+static func format_llama3(messages: Array) -> String:
+	var res = ""
+
+	for i in range(messages.size()):
+		match messages[i]:
+			{"text": var text, "sender": var sender}:
+				res += """<|start_header_id|>%s<|end_header_id|>
+
+%s<|eot_id|>
+""" % [sender, text]
+			_:
+				printerr("Invalid message at index ", i)
+
+	res += "<|start_header_id|>assistant<|end_header_id|>\n\n"
+	return res
+
+static func format_phi3(messages: Array) -> String:
+	var res = ""
+
+	for i in range(messages.size()):
+		match messages[i]:
+			{"text": var text, "sender": var sender}:
+				res +="<|%s|>\n%s<|end|>\n" % [sender, text]
+			_:
+				printerr("Invalid message at index ", i)
+	res += "<|assistant|>\n"
+	return res
+
+static func format_mistral(messages: Array) -> String:
+	var res = ""
+
+	for i in range(messages.size()):
+		match messages[i]:
+			{"text": var text, "sender": var sender}:
+				if sender == "user":
+					res += "[INST] %s [/INST]" % text
+				else:
+					res += "%s</s>"
+			_:
+				printerr("Invalid message at index ", i)
+
+	return res

godot/addons/godot-llama-cpp/plugin.cfg

@@ -4,4 +4,4 @@ name="godot-llama-cpp"
 description="Run large language models in Godot. Powered by llama.cpp."
 author="hazelnutcloud"
 version="0.0.1"
-script="godot-llama-cpp.gd"
+script="plugin.gd"

godot/addons/godot-llama-cpp/godot-llama-cpp.gd → godot/addons/godot-llama-cpp/plugin.gd

@@ -4,9 +4,9 @@ extends EditorPlugin
 
 func _enter_tree():
 	# Initialization of the plugin goes here.
-	add_autoload_singleton("__LlamaBackend", "res://addons/godot-llama-cpp/autoloads/llama-backend.gd")
+  pass
 
 
 func _exit_tree():
 	# Clean-up of the plugin goes here.
-	remove_autoload_singleton("__LlamaBackend")
+  pass

godot/addons/godot-llama-cpp/godot-llama-cpp.gdextension → godot/addons/godot-llama-cpp/plugin.gdextension

@@ -5,8 +5,8 @@ compatibility_minimum = "4.2"
 
 [libraries]
 
-macos.debug = "res://addons/godot-llama-cpp/lib/libgodot-llama-cpp-aarch64-macos-none-Debug.dylib"
-macos.release = "res://addons/godot-llama-cpp/lib/libgodot-llama-cpp-aarch64-macos-none-ReleaseFast.dylib"
+macos.debug = "res://addons/godot-llama-cpp/lib/libgodot-llama-cpp-aarch64-macos-none-ReleaseSafe.dylib"
+macos.release = "res://addons/godot-llama-cpp/lib/libgodot-llama-cpp-aarch64-macos-none-ReleaseSafe.dylib"
 windows.debug.x86_32 = "res://addons/godot-llama-cpp/lib/libgodot-llama-cpp.windows.template_debug.x86_32.dll"
 windows.release.x86_32 = "res://addons/godot-llama-cpp/lib/libgodot-llama-cpp.windows.template_release.x86_32.dll"
 windows.debug.x86_64 = "res://addons/godot-llama-cpp/lib/godot-llama-cpp-x86_64-windows-gnu-Debug.dll"

godot/examples/simple/TextEdit.gd

@@ -0,0 +1,20 @@
+extends TextEdit
+
+signal submit(input: String)
+
+func _gui_input(event: InputEvent) -> void:
+	if event is InputEventKey:
+		var keycode = event.get_keycode_with_modifiers()
+		if keycode == KEY_ENTER and event.is_pressed():
+			handle_submit()
+			accept_event()
+		if keycode == KEY_ENTER | KEY_MASK_SHIFT and event.is_pressed():
+			insert_text_at_caret("\n")
+			accept_event()
+
+func _on_button_pressed() -> void:
+	handle_submit()
+
+func handle_submit() -> void:
+	submit.emit(text)
+	text = ""

godot/examples/simple/form.gd

@@ -0,0 +1,6 @@
+extends HBoxContainer
+
+@onready var text_edit = %TextEdit
+
+func _on_button_pressed() -> void:
+	text_edit.handle_submit()

godot/examples/simple/message.gd

@@ -0,0 +1,23 @@
+class_name Message
+extends Node
+
+@onready var text_container = %Text
+@onready var icon = %Panel
+@export_enum("user", "assistant") var sender: String
+@export var include_in_prompt: bool = true
+var text:
+	get:
+		return text_container.text
+	set(value):
+		text_container.text = value
+
+var completion_id: int = -1
+var pending: bool = false
+var errored: bool = false
+
+func set_text(new_text: String):
+	text_container.text = new_text
+
+func append_text(new_text: String):
+	text_container.text += new_text
+

godot/examples/simple/message.tscn

@@ -0,0 +1,37 @@
+[gd_scene load_steps=5 format=3 uid="uid://t862t0v8ht2q"]
+
+[ext_resource type="Script" path="res://examples/simple/message.gd" id="1_pko33"]
+[ext_resource type="Texture2D" uid="uid://dplw232htshgc" path="res://addons/godot-llama-cpp/assets/godot-llama-cpp-1024x1024.svg" id="2_dvc7y"]
+
+[sub_resource type="StyleBoxTexture" id="StyleBoxTexture_t8bgj"]
+texture = ExtResource("2_dvc7y")
+
+[sub_resource type="Theme" id="Theme_bw3pb"]
+Panel/styles/panel = SubResource("StyleBoxTexture_t8bgj")
+
+[node name="RichTextLabel" type="HBoxContainer"]
+anchors_preset = 15
+anchor_right = 1.0
+anchor_bottom = 1.0
+grow_horizontal = 2
+grow_vertical = 2
+size_flags_horizontal = 3
+theme_override_constants/separation = 20
+script = ExtResource("1_pko33")
+sender = "assistant"
+
+[node name="Panel" type="Panel" parent="."]
+unique_name_in_owner = true
+custom_minimum_size = Vector2(80, 80)
+layout_mode = 2
+size_flags_vertical = 0
+theme = SubResource("Theme_bw3pb")
+
+[node name="Text" type="RichTextLabel" parent="."]
+unique_name_in_owner = true
+layout_mode = 2
+size_flags_horizontal = 3
+focus_mode = 2
+text = "..."
+fit_content = true
+selection_enabled = true

godot/examples/simple/simple.gd

@@ -0,0 +1,53 @@
+extends Node
+
+const message = preload("res://examples/simple/message.tscn")
+
+@onready var messages_container = %MessagesContainer
+@onready var llama_context = %LlamaContext
+
+func _on_text_edit_submit(input: String) -> void:
+	handle_input(input)
+
+func handle_input(input: String) -> void:
+	#var messages = [{ "sender": "system", "text": "You are a pirate chatbot who always responds in pirate speak!" }]
+
+	#var messages = [{ "sender": "system", "text": "You are a helpful chatbot assistant!" }]
+	var messages = []
+	messages.append_array(messages_container.get_children().filter(func(msg: Message): return msg.include_in_prompt).map(
+		func(msg: Message) -> Dictionary:
+			return { "text": msg.text, "sender": msg.sender }
+	))
+	messages.append({"text": input, "sender": "user"})
+	var prompt = ChatFormatter.apply("llama3", messages)
+	print("prompt: ", prompt)
+
+	var completion_id = llama_context.request_completion(prompt)
+
+	var user_message: Message = message.instantiate()
+	messages_container.add_child(user_message)
+	user_message.set_text(input)
+	user_message.sender = "user"
+	user_message.completion_id = completion_id
+
+	var ai_message: Message = message.instantiate()
+	messages_container.add_child(ai_message)
+	ai_message.sender = "assistant"
+	ai_message.completion_id = completion_id
+	ai_message.pending = true
+	ai_message.grab_focus()
+
+
+
+func _on_llama_context_completion_generated(chunk: Dictionary) -> void:
+	var completion_id = chunk.id
+	for msg: Message in messages_container.get_children():
+		if msg.completion_id != completion_id or msg.sender != "assistant":
+			continue
+		if chunk.has("error"):
+			msg.errored = true
+		elif chunk.has("text"):
+			if msg.pending:
+				msg.pending = false
+				msg.set_text(chunk["text"])
+			else:
+				msg.append_text(chunk["text"])

godot/examples/simple/simple.tscn

@@ -0,0 +1,79 @@
+[gd_scene load_steps=6 format=3 uid="uid://c55kb4qvg6geq"]
+
+[ext_resource type="Texture2D" uid="uid://dplw232htshgc" path="res://addons/godot-llama-cpp/assets/godot-llama-cpp-1024x1024.svg" id="1_gjsev"]
+[ext_resource type="Script" path="res://examples/simple/simple.gd" id="1_sruc3"]
+[ext_resource type="PackedScene" uid="uid://t862t0v8ht2q" path="res://examples/simple/message.tscn" id="2_7iip7"]
+[ext_resource type="Script" path="res://examples/simple/TextEdit.gd" id="2_7usqw"]
+[ext_resource type="LlamaModel" path="res://models/meta-llama-3-8b-instruct.Q5_K_M.gguf" id="5_qov1l"]
+
+[node name="Node" type="Node"]
+script = ExtResource("1_sruc3")
+
+[node name="Panel" type="Panel" parent="."]
+anchors_preset = 15
+anchor_right = 1.0
+anchor_bottom = 1.0
+grow_horizontal = 2
+grow_vertical = 2
+
+[node name="MarginContainer" type="MarginContainer" parent="Panel"]
+layout_mode = 1
+anchors_preset = 15
+anchor_right = 1.0
+anchor_bottom = 1.0
+grow_horizontal = 2
+grow_vertical = 2
+theme_override_constants/margin_left = 10
+theme_override_constants/margin_top = 10
+theme_override_constants/margin_right = 10
+theme_override_constants/margin_bottom = 10
+
+[node name="VBoxContainer" type="VBoxContainer" parent="Panel/MarginContainer"]
+layout_mode = 2
+
+[node name="ScrollContainer" type="ScrollContainer" parent="Panel/MarginContainer/VBoxContainer"]
+layout_mode = 2
+size_flags_vertical = 3
+follow_focus = true
+
+[node name="MessagesContainer" type="VBoxContainer" parent="Panel/MarginContainer/VBoxContainer/ScrollContainer"]
+unique_name_in_owner = true
+layout_mode = 2
+size_flags_horizontal = 3
+size_flags_vertical = 3
+theme_override_constants/separation = 30
+
+[node name="RichTextLabel2" parent="Panel/MarginContainer/VBoxContainer/ScrollContainer/MessagesContainer" instance=ExtResource("2_7iip7")]
+layout_mode = 2
+include_in_prompt = false
+
+[node name="Text" parent="Panel/MarginContainer/VBoxContainer/ScrollContainer/MessagesContainer/RichTextLabel2" index="1"]
+text = "How can I help you?"
+
+[node name="HBoxContainer" type="HBoxContainer" parent="Panel/MarginContainer/VBoxContainer"]
+layout_mode = 2
+
+[node name="TextEdit" type="TextEdit" parent="Panel/MarginContainer/VBoxContainer/HBoxContainer"]
+custom_minimum_size = Vector2(2.08165e-12, 100)
+layout_mode = 2
+size_flags_horizontal = 3
+placeholder_text = "Ask me anything..."
+wrap_mode = 1
+script = ExtResource("2_7usqw")
+
+[node name="Button" type="Button" parent="Panel/MarginContainer/VBoxContainer/HBoxContainer"]
+custom_minimum_size = Vector2(100, 2.08165e-12)
+layout_mode = 2
+icon = ExtResource("1_gjsev")
+expand_icon = true
+
+[node name="LlamaContext" type="LlamaContext" parent="."]
+model = ExtResource("5_qov1l")
+temperature = 0.9
+unique_name_in_owner = true
+
+[connection signal="submit" from="Panel/MarginContainer/VBoxContainer/HBoxContainer/TextEdit" to="." method="_on_text_edit_submit"]
+[connection signal="pressed" from="Panel/MarginContainer/VBoxContainer/HBoxContainer/Button" to="Panel/MarginContainer/VBoxContainer/HBoxContainer/TextEdit" method="_on_button_pressed"]
+[connection signal="completion_generated" from="LlamaContext" to="." method="_on_llama_context_completion_generated"]
+
+[editable path="Panel/MarginContainer/VBoxContainer/ScrollContainer/MessagesContainer/RichTextLabel2"]

godot/icon.svg.import

@@ -0,0 +1,37 @@
+[remap]
+
+importer="texture"
+type="CompressedTexture2D"
+uid="uid://beeg0oqle7bnk"
+path="res://.godot/imported/icon.svg-218a8f2b3041327d8a5756f3a245f83b.ctex"
+metadata={
+"vram_texture": false
+}
+
+[deps]
+
+source_file="res://icon.svg"
+dest_files=["res://.godot/imported/icon.svg-218a8f2b3041327d8a5756f3a245f83b.ctex"]
+
+[params]
+
+compress/mode=0
+compress/high_quality=false
+compress/lossy_quality=0.7
+compress/hdr_compression=1
+compress/normal_map=0
+compress/channel_pack=0
+mipmaps/generate=false
+mipmaps/limit=-1
+roughness/mode=0
+roughness/src_normal=""
+process/fix_alpha_border=true
+process/premult_alpha=false
+process/normal_map_invert_y=false
+process/hdr_as_srgb=false
+process/hdr_clamp_exposure=false
+process/size_limit=0
+detect_3d/compress_to=1
+svg/scale=1.0
+editor/scale_with_editor_scale=false
+editor/convert_colors_with_editor_theme=false