Merge pull request #323 from o-on-x/o-on-x-wip-tweet-split

utils.ts example tweet splitting

Author: lalalune

packages/client-twitter/src/utils.ts

@@ -1,11 +1,13 @@
+// utils.ts
+
 import { Tweet } from "agent-twitter-client";
 import { embeddingZeroVector } from "@ai16z/eliza/src/memory.ts";
 import { Content, Memory, UUID } from "@ai16z/eliza/src/types.ts";
 import { stringToUuid } from "@ai16z/eliza/src/uuid.ts";
 import { ClientBase } from "./base.ts";
 import { elizaLogger } from "@ai16z/eliza/src/logger.ts";
 
-const MAX_TWEET_LENGTH = 240;
+const MAX_TWEET_LENGTH = 280; // Updated to Twitter's current character limit
 
 export const wait = (minTime: number = 1000, maxTime: number = 3000) => {
     const waitTime =
@@ -17,13 +19,13 @@ export const isValidTweet = (tweet: Tweet): boolean => {
     // Filter out tweets with too many hashtags, @s, or $ signs, probably spam or garbage
     const hashtagCount = (tweet.text?.match(/#/g) || []).length;
     const atCount = (tweet.text?.match(/@/g) || []).length;
-    const dollarSignCount = tweet.text?.match(/\$/g) || [];
-    const totalCount = hashtagCount + atCount + dollarSignCount.length;
+    const dollarSignCount = (tweet.text?.match(/\$/g) || []).length;
+    const totalCount = hashtagCount + atCount + dollarSignCount;
 
     return (
         hashtagCount <= 1 &&
         atCount <= 2 &&
-        dollarSignCount.length <= 1 &&
+        dollarSignCount <= 1 &&
         totalCount <= 3
     );
 };
@@ -40,7 +42,7 @@ export async function buildConversationThread(
             elizaLogger.log("No current tweet found");
             return;
         }
-        // check if the current tweet has already been saved
+        // Check if the current tweet has already been saved
         const memory = await client.runtime.messageManager.getMemoryById(
             stringToUuid(currentTweet.id + "-" + client.runtime.agentId)
         );
@@ -49,7 +51,10 @@ export async function buildConversationThread(
             const roomId = stringToUuid(
                 currentTweet.conversationId + "-" + client.runtime.agentId
             );
-            const userId = stringToUuid(currentTweet.userId);
+            const userId =
+                currentTweet.userId === client.twitterUserId
+                    ? client.runtime.agentId
+                    : stringToUuid(currentTweet.userId);
 
             await client.runtime.ensureConnection(
                 userId,
@@ -59,11 +64,12 @@ export async function buildConversationThread(
                 "twitter"
             );
 
-            client.runtime.messageManager.createMemory({
+            await client.runtime.messageManager.createMemory({
                 id: stringToUuid(
                     currentTweet.id + "-" + client.runtime.agentId
                 ),
                 agentId: client.runtime.agentId,
+                userId: userId,
                 content: {
                     text: currentTweet.text,
                     source: "twitter",
@@ -78,10 +84,6 @@ export async function buildConversationThread(
                 },
                 createdAt: currentTweet.timestamp * 1000,
                 roomId,
-                userId:
-                    currentTweet.userId === client.twitterUserId
-                        ? client.runtime.agentId
-                        : stringToUuid(currentTweet.userId),
                 embedding: embeddingZeroVector,
             });
         }
@@ -100,7 +102,7 @@ export async function buildConversationThread(
     await processThread(tweet);
 }
 
-export async function sendTweetChunks(
+export async function sendTweet(
     client: ClientBase,
     content: Content,
     roomId: UUID,
@@ -109,25 +111,26 @@ export async function sendTweetChunks(
 ): Promise<Memory[]> {
     const tweetChunks = splitTweetContent(content.text);
     const sentTweets: Tweet[] = [];
+    let previousTweetId = inReplyTo;
 
     for (const chunk of tweetChunks) {
         const result = await client.requestQueue.add(
             async () =>
                 await client.twitterClient.sendTweet(
-                    chunk.replaceAll(/\\n/g, "\n").trim(),
-                    inReplyTo
+                    chunk.trim(),
+                    previousTweetId
                 )
         );
-        // console.log("send tweet result:\n", result);
+        // Parse the response
         const body = await result.json();
-        console.log("send tweet body:\n", body.data.create_tweet.tweet_results);
         const tweetResult = body.data.create_tweet.tweet_results.result;
 
-        const finalTweet = {
+        const finalTweet: Tweet = {
             id: tweetResult.rest_id,
             text: tweetResult.legacy.full_text,
             conversationId: tweetResult.legacy.conversation_id_str,
-            createdAt: tweetResult.legacy.created_at,
+            //createdAt: 
+            timestamp: tweetResult.timestamp * 1000,
             userId: tweetResult.legacy.user_id_str,
             inReplyToStatusId: tweetResult.legacy.in_reply_to_status_id_str,
             permanentUrl: `https://twitter.com/${twitterUsername}/status/${tweetResult.rest_id}`,
@@ -137,72 +140,14 @@ export async function sendTweetChunks(
             thread: [],
             urls: [],
             videos: [],
-        } as Tweet;
+        };
 
         sentTweets.push(finalTweet);
-    }
-
-    const memories: Memory[] = sentTweets.map((tweet) => ({
-        id: stringToUuid(tweet.id + "-" + client.runtime.agentId),
-        agentId: client.runtime.agentId,
-        userId: client.runtime.agentId,
-        content: {
-            text: tweet.text,
-            source: "twitter",
-            url: tweet.permanentUrl,
-            inReplyTo: tweet.inReplyToStatusId
-                ? stringToUuid(
-                      tweet.inReplyToStatusId + "-" + client.runtime.agentId
-                  )
-                : undefined,
-        },
-        roomId,
-        embedding: embeddingZeroVector,
-        createdAt: tweet.timestamp * 1000,
-    }));
-
-    return memories;
-}
+        previousTweetId = finalTweet.id;
 
-export async function sendTweet(
-    client: ClientBase,
-    content: Content,
-    roomId: UUID,
-    twitterUsername: string,
-    inReplyTo: string
-): Promise<Memory[]> {
-    const chunk = truncateTweetContent(content.text);
-    const sentTweets: Tweet[] = [];
-
-    const result = await client.requestQueue.add(
-        async () =>
-            await client.twitterClient.sendTweet(
-                chunk.replaceAll(/\\n/g, "\n").trim(),
-                inReplyTo
-            )
-    );
-    // console.log("send tweet result:\n", result);
-    const body = await result.json();
-    console.log("send tweet body:\n", body.data.create_tweet.tweet_results);
-    const tweetResult = body.data.create_tweet.tweet_results.result;
-
-    const finalTweet = {
-        id: tweetResult.rest_id,
-        text: tweetResult.legacy.full_text,
-        conversationId: tweetResult.legacy.conversation_id_str,
-        createdAt: tweetResult.legacy.created_at,
-        userId: tweetResult.legacy.user_id_str,
-        inReplyToStatusId: tweetResult.legacy.in_reply_to_status_id_str,
-        permanentUrl: `https://twitter.com/${twitterUsername}/status/${tweetResult.rest_id}`,
-        hashtags: [],
-        mentions: [],
-        photos: [],
-        thread: [],
-        urls: [],
-        videos: [],
-    } as Tweet;
-
-    sentTweets.push(finalTweet);
+        // Wait a bit between tweets to avoid rate limiting issues
+        await wait(1000, 2000);
+    }
 
     const memories: Memory[] = sentTweets.map((tweet) => ({
         id: stringToUuid(tweet.id + "-" + client.runtime.agentId),
@@ -227,56 +172,85 @@ export async function sendTweet(
 }
 
 function splitTweetContent(content: string): string[] {
-    const tweetChunks: string[] = [];
-    let currentChunk = "";
+    const maxLength = MAX_TWEET_LENGTH;
+    const paragraphs = content.split("\n\n").map((p) => p.trim());
+    const tweets: string[] = [];
+    let currentTweet = "";
+
+    for (const paragraph of paragraphs) {
+        if (!paragraph) continue;
 
-    const words = content.split(" ");
-    for (const word of words) {
-        if (currentChunk.length + word.length + 1 <= MAX_TWEET_LENGTH) {
-            currentChunk += (currentChunk ? " " : "") + word;
+        if ((currentTweet + "\n\n" + paragraph).trim().length <= maxLength) {
+            if (currentTweet) {
+                currentTweet += "\n\n" + paragraph;
+            } else {
+                currentTweet = paragraph;
+            }
         } else {
-            tweetChunks.push(currentChunk);
-            currentChunk = word;
+            if (currentTweet) {
+                tweets.push(currentTweet.trim());
+            }
+            if (paragraph.length <= maxLength) {
+                currentTweet = paragraph;
+            } else {
+                // Split long paragraph into smaller chunks
+                const chunks = splitParagraph(paragraph, maxLength);
+                tweets.push(...chunks.slice(0, -1));
+                currentTweet = chunks[chunks.length - 1];
+            }
         }
     }
 
-    if (currentChunk) {
-        tweetChunks.push(currentChunk);
+    if (currentTweet) {
+        tweets.push(currentTweet.trim());
     }
 
-    return tweetChunks;
+    return tweets;
 }
 
-export function truncateTweetContent(content: string): string {
-    // if its 240, delete the last line
-    if (content.length === MAX_TWEET_LENGTH) {
-        return content.slice(0, content.lastIndexOf("\n"));
-    }
+function splitParagraph(paragraph: string, maxLength: number): string[] {
+    const sentences = paragraph.match(/[^\.!\?]+[\.!\?]+|[^\.!\?]+$/g) || [paragraph];
+    const chunks: string[] = [];
+    let currentChunk = "";
 
-    // if its still bigger than 240, delete everything after the last period
-    if (content.length > MAX_TWEET_LENGTH) {
-        return content.slice(0, content.lastIndexOf("."));
+    for (const sentence of sentences) {
+        if ((currentChunk + " " + sentence).trim().length <= maxLength) {
+            if (currentChunk) {
+                currentChunk += " " + sentence;
+            } else {
+                currentChunk = sentence;
+            }
+        } else {
+            if (currentChunk) {
+                chunks.push(currentChunk.trim());
+            }
+            if (sentence.length <= maxLength) {
+                currentChunk = sentence;
+            } else {
+                // Split long sentence into smaller pieces
+                const words = sentence.split(" ");
+                currentChunk = "";
+                for (const word of words) {
+                    if ((currentChunk + " " + word).trim().length <= maxLength) {
+                        if (currentChunk) {
+                            currentChunk += " " + word;
+                        } else {
+                            currentChunk = word;
+                        }
+                    } else {
+                        if (currentChunk) {
+                            chunks.push(currentChunk.trim());
+                        }
+                        currentChunk = word;
+                    }
+                }
+            }
+        }
     }
 
-    // while its STILL bigger than 240, find the second to last exclamation point or period and delete everything after it
-    let iterations = 0;
-    while (content.length > MAX_TWEET_LENGTH && iterations < 10) {
-        iterations++;
-        // second to last index of period or exclamation point
-        const secondToLastIndexOfPeriod = content.lastIndexOf(
-            ".",
-            content.length - 2
-        );
-        const secondToLastIndexOfExclamation = content.lastIndexOf(
-            "!",
-            content.length - 2
-        );
-        const secondToLastIndex = Math.max(
-            secondToLastIndexOfPeriod,
-            secondToLastIndexOfExclamation
-        );
-        content = content.slice(0, secondToLastIndex);
+    if (currentChunk) {
+        chunks.push(currentChunk.trim());
     }
 
-    return content;
+    return chunks;
 }