Merge pull request #671 from cygaar/fix_url_embeddings

fix: embeddings for messages with urls

Author: shakkernerd

packages/core/src/knowledge.ts

@@ -94,6 +94,10 @@ export function preprocess(content: string): string {
             .replace(/!\[(.*?)\]\(.*?\)/g, "$1")
             // Remove links but keep text
             .replace(/\[(.*?)\]\(.*?\)/g, "$1")
+            // Simplify URLs: remove protocol and simplify to domain+path
+            .replace(/(https?:\/\/)?(www\.)?([^\s]+\.[^\s]+)/g, "$3")
+            // Remove Discord mentions specifically
+            .replace(/<@[!&]?\d+>/g, "")
             // Remove HTML tags
             .replace(/<[^>]*>/g, "")
             // Remove horizontal rules
@@ -105,10 +109,8 @@ export function preprocess(content: string): string {
             .replace(/\s+/g, " ")
             // Remove multiple newlines
             .replace(/\n{3,}/g, "\n\n")
-            // strip all special characters
-            .replace(/[^a-zA-Z0-9\s]/g, "")
-            // Remove Discord mentions
-            .replace(/<@!?\d+>/g, "")
+            // Remove special characters except those common in URLs
+            .replace(/[^a-zA-Z0-9\s\-_./:?=&]/g, "")
             .trim()
             .toLowerCase()
     );