Merge pull request #2730 from elizaOS/tcm-improve-twitter-post

feat: improve twitter parsing

Author: tcm390

packages/client-twitter/src/post.ts

@@ -10,6 +10,7 @@ import {
     type UUID,
     truncateToCompleteSentence,
     parseJSONObjectFromText,
+    extractAttributes,
 } from "@elizaos/core";
 import { elizaLogger } from "@elizaos/core";
 import type { ClientBase } from "./base.ts";
@@ -463,6 +464,22 @@ export class TwitterPostClient {
         }
     }
 
+    /**
+     * Cleans a JSON-like response string by removing unnecessary markers, line breaks, and extra whitespace.
+     * This is useful for handling improperly formatted JSON responses from external sources.
+     *
+     * @param response - The raw JSON-like string response to clean.
+     * @returns The cleaned string, ready for parsing or further processing.
+     */
+
+    cleanJsonResponse(response: string): string {
+        return response
+            .replace(/```json\s*/g, "") // Remove ```json
+            .replace(/```\s*/g, "") // Remove any remaining ```
+            .replace(/(\r\n|\n|\r)/g, "") // Remove line breaks
+            .trim();
+    }
+
     /**
      * Generates and posts a new tweet. If isDryRun is true, only logs what would have been posted.
      */
@@ -512,11 +529,7 @@ export class TwitterPostClient {
                 modelClass: ModelClass.SMALL,
             });
 
-            const newTweetContent = response
-                .replace(/```json\s*/g, "") // Remove ```json
-                .replace(/```\s*/g, "") // Remove any remaining ```
-                .replace(/(\r\n|\n|\r)/g, "") // Remove line break
-                .trim();
+            const newTweetContent = this.cleanJsonResponse(response);
 
             // First attempt to clean content
             let cleanedContent = "";
@@ -544,6 +557,13 @@ export class TwitterPostClient {
                     .trim();
             }
 
+            if (!cleanedContent) {
+                cleanedContent = truncateToCompleteSentence(
+                    extractAttributes(newTweetContent, ["text"]).text,
+                    this.client.twitterConfig.MAX_TWEET_LENGTH,
+                );
+            }
+
             if (!cleanedContent) {
                 elizaLogger.error(
                     "Failed to extract valid content from response:",
@@ -634,25 +654,29 @@ export class TwitterPostClient {
         elizaLogger.log("generate tweet content response:\n" + response);
 
         // First clean up any markdown and newlines
-        const cleanedResponse = response
-            .replace(/```json\s*/g, "") // Remove ```json
-            .replace(/```\s*/g, "") // Remove any remaining ```
-            .replace(/(\r\n|\n|\r)/g, "") // Remove line break
-            .trim();
+        const cleanedResponse = this.cleanJsonResponse(response);
 
         // Try to parse as JSON first
         try {
             const jsonResponse = parseJSONObjectFromText(cleanedResponse);
             if (jsonResponse.text) {
-                return this.trimTweetLength(jsonResponse.text);
+                const truncateContent = truncateToCompleteSentence(
+                    jsonResponse.text,
+                    this.client.twitterConfig.MAX_TWEET_LENGTH,
+                );
+                return truncateContent;
             }
             if (typeof jsonResponse === "object") {
                 const possibleContent =
                     jsonResponse.content ||
                     jsonResponse.message ||
                     jsonResponse.response;
                 if (possibleContent) {
-                    return this.trimTweetLength(possibleContent);
+                    const truncateContent = truncateToCompleteSentence(
+                        possibleContent,
+                        this.client.twitterConfig.MAX_TWEET_LENGTH,
+                    );
+                    return truncateContent;
                 }
             }
         } catch (error) {
@@ -664,24 +688,21 @@ export class TwitterPostClient {
                 response,
             );
         }
-        // If not JSON or no valid content found, clean the raw text
-        return this.trimTweetLength(cleanedResponse);
-    }
 
-    // Helper method to ensure tweet length compliance
-    private trimTweetLength(text: string, maxLength = 280): string {
-        if (text.length <= maxLength) return text;
+        let truncateContent = truncateToCompleteSentence(
+            extractAttributes(cleanedResponse, ["text"]).text,
+            this.client.twitterConfig.MAX_TWEET_LENGTH,
+        );
 
-        // Try to cut at last sentence
-        const lastSentence = text.slice(0, maxLength).lastIndexOf(".");
-        if (lastSentence > 0) {
-            return text.slice(0, lastSentence + 1).trim();
+        if (!truncateContent) {
+            // If not JSON or no valid content found, clean the raw text
+            truncateContent = truncateToCompleteSentence(
+                cleanedResponse,
+                this.client.twitterConfig.MAX_TWEET_LENGTH,
+            );
         }
 
-        // Fallback to word boundary
-        return (
-            text.slice(0, text.lastIndexOf(" ", maxLength - 3)).trim() + "..."
-        );
+        return truncateContent;
     }
 
     /**

packages/core/src/parsing.ts

@@ -12,7 +12,7 @@ If {{agentName}} is talking too much, you can choose [IGNORE]
 Your response must include one of the options.`;
 
 export const parseShouldRespondFromText = (
-    text: string
+    text: string,
 ): "RESPOND" | "IGNORE" | "STOP" | null => {
     const match = text
         .split("\n")[0]
@@ -92,6 +92,7 @@ export function parseJsonArrayFromText(text: string) {
             jsonData = JSON.parse(normalizedJson);
         } catch (e) {
             console.error("Error parsing JSON:", e);
+            console.error("Text is not JSON", text);
         }
     }
 
@@ -106,6 +107,7 @@ export function parseJsonArrayFromText(text: string) {
                 const normalizedJson = arrayMatch[0].replace(/'/g, '"');
                 jsonData = JSON.parse(normalizedJson);
             } catch (e) {
+                console.error("Text is not JSON", text);
                 console.error("Error parsing JSON:", e);
             }
         }
@@ -129,7 +131,7 @@ export function parseJsonArrayFromText(text: string) {
  * @returns An object parsed from the JSON string if successful; otherwise, null or the result of parsing an array.
  */
 export function parseJSONObjectFromText(
-    text: string
+    text: string,
 ): Record<string, any> | null {
     let jsonData = null;
 
@@ -140,6 +142,7 @@ export function parseJSONObjectFromText(
             jsonData = JSON.parse(jsonBlockMatch[1]);
         } catch (e) {
             console.error("Error parsing JSON:", e);
+            console.error("Text is not JSON", text);
             return null;
         }
     } else {
@@ -151,6 +154,7 @@ export function parseJSONObjectFromText(
                 jsonData = JSON.parse(objectMatch[0]);
             } catch (e) {
                 console.error("Error parsing JSON:", e);
+                console.error("Text is not JSON", text);
                 return null;
             }
         }
@@ -169,10 +173,34 @@ export function parseJSONObjectFromText(
     }
 }
 
+/**
+ * Extracts specific attributes (e.g., user, text, action) from a JSON-like string using regex.
+ * @param response - The cleaned string response to extract attributes from.
+ * @param attributesToExtract - An array of attribute names to extract.
+ * @returns An object containing the extracted attributes.
+ */
+export function extractAttributes(
+    response: string,
+    attributesToExtract: string[],
+): { [key: string]: string | undefined } {
+    const attributes: { [key: string]: string | undefined } = {};
+
+    attributesToExtract.forEach((attribute) => {
+        const match = response.match(
+            new RegExp(`"${attribute}"\\s*:\\s*"([^"]*)"`, "i"),
+        );
+        if (match) {
+            attributes[attribute] = match[1];
+        }
+    });
+
+    return attributes;
+}
+
 export const postActionResponseFooter = `Choose any combination of [LIKE], [RETWEET], [QUOTE], and [REPLY] that are appropriate. Each action must be on its own line. Your response must only include the chosen actions.`;
 
 export const parseActionResponseFromText = (
-    text: string
+    text: string,
 ): { actions: ActionResponse } => {
     const actions: ActionResponse = {
         like: false,
@@ -211,7 +239,7 @@ export const parseActionResponseFromText = (
  */
 export function truncateToCompleteSentence(
     text: string,
-    maxLength: number
+    maxLength: number,
 ): string {
     if (text.length <= maxLength) {
         return text;