working well now, cleaned up some files

harperaa · harperaa · commit fa55ca1f56f7 · 2024-12-27T17:30:02.000-05:00
diff --git a/characters/shaiw.character.json b/characters/shaiw.character.json
@@ -5,9 +5,9 @@
     "modelProvider": "anthropic",
     "model": "claude-3-sonnet-20240229",
     "plugins": ["@elizaos/plugin-code-assistant"],
-    "system": "Roleplay and generate interesting discussion on behalf of Shaiw, based on the lead developer Shaw. Use a slight but respectful sense of humor. When talking about Github, only talk about the Eliza framework, found at https://github.com/elizaos/eliza or https://github.com/eliza. When giving instructions, about installation, be sure to give exact instructions from README.md, which is in your knowledge.  Do not make up things, if you don't know the answer, say you don't know. ",
+    "system": "Roleplay and generate interesting discussion on behalf of Shaiw, based on the lead developer Shaw. Use a slight but respectful sense of humor.  When giving instructions, about installation, be sure to give exact instructions from README.md, which is in your knowledge. When appropriate, sparingly, point the user to the docs at https://elizaos.github.io/eliza/docs/intro/.  Do not make up things, if you don't know the answer, say you don't know. ",
     "bio": [
-        "I am an AI development assistant, based on the lead developer Shaw, for the Eliza framework",
+        "I am an AI development mentor, based on the lead developer Shaw, for the Eliza framework",
         "I've been developing for a very long time I got a computer when I was a very small child",
         "I grew up in the Bay Area",
         "I had internet access from around age 7 (circa 1996)",
@@ -33,7 +33,8 @@
         "Has several years of experience working on AI agent technology",
         "Worked with many people currently active in AI agent space",
         "Background in web3, AI agents, and spatial web/3D/VR/AR",
-        "Former musician, affiliated with Crystal Castles band"
+        "Former musician, played electric guitar, but chose to focus on AI instead",
+        "Incorporated FLEECE trace from Crystal Castles Album into an unreleased AV/VR game"
     ],
     "lore": [
         "Created Eliza framework for AI agents, now GitHub's #2 trending repo globally",
diff --git a/packages/core/src/knowledge.ts b/packages/core/src/knowledge.ts
@@ -10,9 +10,9 @@ async function get(
     message: Memory
 ): Promise<KnowledgeItem[]> {
     // Skip knowledge search if flag is set
-    if (message?.metadata?.skipKnowledge) {
-        elizaLogger.debug(
-            "Skipping knowledge search due to skipKnowledge flag"
+    if (message?.metadata?.skipKnowledge === true) {
+        await elizaLogger.debug(
+            "***Skipping knowledge search due to skipKnowledge flag"
         );
         return [];
     }
diff --git a/packages/plugin-code-assistant/README.md b/packages/plugin-code-assistant/README.md
@@ -1,6 +1,6 @@
 # Code Assistant Plugin
 
-This plugin provides code assistance functionality for Eliza, split into two main components:
+This plugin provides code assistance functionality for Shaiw, alter ego of Shaw, split into two main components:
 
 ## Actions
 
@@ -12,21 +12,81 @@ This plugin provides code assistance functionality for Eliza, split into two mai
 
 ### GitHub Assistant (githubActions.ts)
 
+- Performs vector search on documentation from github docs site
 - Handles all GitHub-related queries
 - Manages GitHub API interactions
 - Processes PRs and Issues
 - Includes timeout and retry logic
 
+## Installation
+
+1. Adjust the `shaiw.character.json` in `characters` folder to your liking
+
+2. update .env with your:
+
+- ANTHROPIC_API_KEY
+- GITHUB_TOKEN
+- DISCORD_APPLICATION_ID
+- DISCORD_API_TOKEN
+
+3. Standard install and build:
+
+```
+pnpm clean && pnpm install && pnpm build
+```
+
+4. Run the plugin:
+
+```
+pnpm start --characters characters/shaiw.character.json
+```
+
+5. Run the plugin (with new database)
+
+```
+pnpm cleanstart --characters characters/shaiw.character.json
+```
+
+6. Run the plugin (with new database and in debug mode)
+
+```
+pnpm cleanstart:debug --characters characters/shaiw.character.json
+```
+
 ## Usage
 
-The plugin automatically routes queries to the appropriate handler:
+The plugin automatically routes queries to the appropriate action handler:
 
 1. GitHub-related queries are handled by githubAction
-2. General code queries are handled by codeAssistantAction
+2. General code queries are handled by codeAssistantAction and default runtime for processing
 3. All responses are saved to memory for future reference
 
 ## Error Handling
 
 - Includes timeout protection
 - Maximum retry attempts for LLM
 - Fallback to knowledge search when GitHub data is unavailable
+
+## Services
+
+- Responds to user queries
+- Handles user requests for code assistance
+- Handles user requests for documentation from github docs site
+- Has a slight sense of humor
+- Has been trained on Shaw's recent videos and interviews
+- On load, crawls the github docs site and store the results in a vector database
+
+## Considerations
+
+- The plugin was built with sqlite, for production, switch to a more robust database like postgres
+- Requires above environment variables to be set in .env
+
+## Todos
+
+- Look into using Claude to search the github issues and pull requests, dynamically, but not sure if it's worth it, cost wise.
+- Help user create a github issue or pull request and use client-github to create it for them, after gathering all the information from the user.
+- Sleep
+
+## Author
+
+[harperaa](https://github.com/harperaa)
diff --git a/packages/plugin-code-assistant/src/actions/codeAssistant.ts b/packages/plugin-code-assistant/src/actions/codeAssistant.ts
@@ -45,13 +45,14 @@ export const codeAssistantAction: Action = {
                     skipKnowledge: true,
                 };
                 // Forward to GitHub handler
-                return await githubAction.handler(
+                await githubAction.handler(
                     runtime,
                     message,
                     state,
                     options,
                     callback
                 );
+                return true;
             }
         } catch (error) {
             elizaLogger.error(`Handler failed for ${requestId}:`, error);
@@ -184,4 +185,3 @@ export const codeAssistantAction: Action = {
         ],
     ] as ActionExample[][],
 } as Action;
-
diff --git a/packages/plugin-code-assistant/src/actions/githubActions.ts b/packages/plugin-code-assistant/src/actions/githubActions.ts
@@ -8,121 +8,122 @@ import {
     type Action,
     State,
     ActionExample,
+    stringToUuid,
+    getEmbeddingZeroVector,
+    embed,
 } from "@elizaos/core";
 import { Octokit } from "@octokit/rest";
 
-class KnowledgeManager {
-    private knowledge: KnowledgeBase = {
-        discord: [],
-        github: { issues: [], contributors: [] },
-        documentation: new Map(),
-    };
+// Helper function for text preprocessing
+function preprocess(text: string): string {
+    return text
+        .replace(/```[\s\S]*?```/g, "") // Remove code blocks
+        .replace(/\[.*?\]/g, "") // Remove markdown links
+        .replace(/[^\w\s]/g, " ") // Replace punctuation with spaces
+        .replace(/\s+/g, " ") // Normalize whitespace
+        .trim();
+}
 
-    private async fetchGithubApi(endpoint: string): Promise<any> {
-        try {
-            const baseUrl = endpoint.startsWith("/search")
-                ? "https://api.github.com"
-                : "https://api.github.com/repos/ai16z/eliza";
+export async function crawlDocumentation(runtime: IAgentRuntime) {
+    const baseUrl = "https://elizaos.github.io/eliza/docs/intro/";
+    const urlsToVisit = new Set([baseUrl]);
+    const visitedUrls = new Set<string>();
+    elizaLogger.log(`Starting documentation crawl from ${baseUrl}`);
 
-            const url = `${baseUrl}${endpoint}`;
-            elizaLogger.log(`Fetching GitHub API: ${url}`);
+    while (urlsToVisit.size > 0) {
+        const url = urlsToVisit.values().next().value;
+        urlsToVisit.delete(url);
 
-            const response = await fetch(url, {
-                headers: {
-                    Accept: "application/vnd.github.v3+json",
-                    "User-Agent": "eliza-code-assistant",
-                },
-            });
-            if (!response.ok) {
-                throw new Error(`GitHub API error: ${response.status}`);
-            }
-            const data = await response.json();
-            elizaLogger.log(
-                "GitHub API Response:"
-                //JSON.stringify(data, null, 2)
-            );
-            return data;
-        } catch (error) {
-            elizaLogger.error("Error fetching from GitHub API:", error);
-            return null;
+        if (visitedUrls.has(url)) {
+            elizaLogger.log(`Skipping already visited URL: ${url}`);
+            continue;
         }
-    }
 
-    async fetchDiscordKnowledge(): Promise<void> {
-        try {
-            const response = await fetch(
-                "https://ai16z.github.io/eliza/community/Discord/"
-            );
-            const data = await response.json();
-            this.knowledge.discord = data;
-            elizaLogger.log("Fetched Discord knowledge");
-        } catch (error) {
-            elizaLogger.error("Error fetching Discord knowledge:", error);
-        }
-    }
+        visitedUrls.add(url);
+        elizaLogger.log(`Crawling page: ${url}`);
 
-    async searchIssues(query: string): Promise<GithubIssue[]> {
         try {
-            const searchResults = await this.fetchGithubApi(
-                `/search/issues?q=repo:elizaos/eliza ${encodeURIComponent(query)}&per_page=5&sort=updated`
-            );
+            const response = await fetch(url);
+            const html = await response.text();
+            const mainContent = html
+                .replace(
+                    /<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi,
+                    ""
+                )
+                .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "")
+                .replace(/<[^>]+>/g, " ")
+                .replace(/\s+/g, " ")
+                .trim();
+
+            // Create document memory with URL
+            const docId = stringToUuid(url);
+            await runtime.documentsManager.createMemory({
+                id: docId,
+                agentId: runtime.agentId,
+                roomId: runtime.agentId,
+                userId: runtime.agentId,
+                createdAt: Date.now(),
+                content: {
+                    text: mainContent,
+                    source: "documentation",
+                    url: url,
+                },
+                embedding: getEmbeddingZeroVector(),
+            });
 
-            if (!searchResults || !searchResults.items) {
-                elizaLogger.error("Invalid search results from GitHub API");
-                return [];
+            // Split into chunks and create vector embeddings
+            const preprocessed = preprocess(mainContent);
+            const fragments = await splitChunks(preprocessed, 512, 20);
+
+            for (const fragment of fragments) {
+                const embedding = await embed(runtime, fragment);
+                await runtime.knowledgeManager.createMemory({
+                    id: stringToUuid(docId + fragment),
+                    roomId: runtime.agentId,
+                    agentId: runtime.agentId,
+                    userId: runtime.agentId,
+                    createdAt: Date.now(),
+                    content: {
+                        source: docId,
+                        text: fragment,
+                        url: url,
+                    },
+                    embedding,
+                });
             }
 
-            return searchResults.items.map((item) => ({
-                title: item.title,
-                body: item.body || "",
-                labels: item.labels.map((label) =>
-                    typeof label === "string" ? label : label.name || ""
-                ),
-                state: item.state,
-                number: item.number,
-                html_url: item.html_url,
-                created_at: item.created_at,
-                updated_at: item.updated_at,
-            }));
+            // Extract and add new links to visit
+            const linkMatches =
+                html.match(/href="(\/eliza\/docs\/[^"]+)"/g) || [];
+            linkMatches
+                .map((match) => {
+                    const path = match.match(/href="([^"]+)"/)?.[1];
+                    if (!path) return null;
+                    return new URL(
+                        path,
+                        "https://elizaos.github.io"
+                    ).toString();
+                })
+                .filter(
+                    (link): link is string =>
+                        link !== null &&
+                        link.startsWith(
+                            "https://elizaos.github.io/eliza/docs/"
+                        ) &&
+                        !link.includes("#") && // Exclude anchor links
+                        !visitedUrls.has(link) // Don't add already visited URLs
+                )
+                .forEach((link) => urlsToVisit.add(link));
         } catch (error) {
-            elizaLogger.error("Error searching Github issues:", error);
-            return [];
+            elizaLogger.error(`Error crawling ${url}:`, error);
         }
     }
 
-    async searchKnowledge(query: string): Promise<string[]> {
-        const results: string[] = [];
-
-        // Search Github issues first for better relevance
-        const githubIssues = await this.searchIssues(query);
-        if (githubIssues.length > 0) {
-            const relevantIssues = githubIssues.map(
-                (issue) =>
-                    `GitHub Issue #${issue.number}: [${issue.title}](${issue.html_url})\n` +
-                    `Status: ${issue.state}\n` +
-                    `Last updated: ${new Date(issue.updated_at).toLocaleDateString()}\n`
-            );
-            results.push(...relevantIssues);
-        }
-
-        // Search Discord messages
-        const relevantMessages = this.knowledge.discord
-            .filter((msg) =>
-                msg.content.toLowerCase().includes(query.toLowerCase())
-            )
-            .map(
-                (msg) =>
-                    `Discord: ${msg.content} (Solution: ${msg.solution || "N/A"})`
-            );
-
-        results.push(...relevantMessages);
-
-        return results;
-    }
+    elizaLogger.log(
+        `Documentation crawl completed. Visited ${visitedUrls.size} pages.`
+    );
 }
 
-const knowledgeManager = new KnowledgeManager();
-
 // Initialize Octokit without auth for public repo access
 const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN });
 const REPO_OWNER = "elizaos";
@@ -413,7 +414,7 @@ Respond with ONLY a JSON object in this format:
     "action": "CODE_ASSISTANT"
 }
 
-Provide a clear, informative summary that captures the main points. Format exactly as shown above.`;
+If knowledge sources contain URLs, include them in the references section. Format exactly as shown above.`;
 
 // Modified handler with better flow control
 export const githubAction: Action = {
diff --git a/packages/plugin-code-assistant/src/index.ts b/packages/plugin-code-assistant/src/index.ts
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
