twitter scraper target

Author: augchan42

.gitignore

@@ -48,3 +48,5 @@ cache/*
 log*.txt
 # Explicitly include plugin-node data files
 !packages/plugin-node/data/**
+
+.turbo

README.md

@@ -144,6 +144,39 @@ docker-compose up -d --build
 - Docker Compose v2.0.0 or higher
 
 
+
+### Twitter Scraper Target
+
+We have included a new script to scrape tweets from a specified Twitter user. This script uses the `agent-twitter-client` package to perform the scraping.
+
+#### Usage
+
+1. Ensure you have the necessary environment variables set in your `.env` file:
+    ```plaintext
+    TWITTER_USERNAME=your_twitter_username
+    TWITTER_PASSWORD=your_twitter_password
+    TWITTER_EMAIL=your_twitter_email
+    ```
+
+2. Run the script with the target Twitter username as an argument:
+    ```bash
+    pnpm scrape-twitter <username>
+    ```
+
+   Replace `<username>` with the Twitter handle of the user you want to scrape tweets from.
+
+3. The scraped tweets will be saved in the `data/tweets` directory with a timestamped filename.
+
+#### Example
+
+To scrape tweets from the user `exampleuser`, you would run:
+    ```bash
+    pnpm scrape-twitter exampleuser
+    ```
+
+    This will create a file named something like `exampleuser_2023-09-15_12-34-56.json` in the `data/tweets` directory, containing the scraped tweets.
+
+
 ### Community & contact
 
 -   [GitHub Issues](https://github.com/ai16z/eliza/issues). Best for: bugs you encounter using Eliza, and feature proposals.

data/discord/rules/temple_page_1.csv

@@ -20,7 +20,8 @@
         "docker:bash": "bash ./scripts/docker.sh bash",
         "docker:start": "bash ./scripts/docker.sh start",
         "docker": "pnpm docker:build && pnpm docker:run && pnpm docker:bash",
-        "test": "pnpm --dir packages/core test"
+        "test": "pnpm --dir packages/core test",
+        "scrape-twitter": "ts-node scripts/twitter-scraper.ts"
     },
     "nodemonConfig": {
         "watch": [

data/discord/rules/temple_page_2.csv

@@ -1,76 +1,14 @@
-import { Scraper } from "agent-twitter-client";
-import dotenv from "dotenv";
 import fs from "fs";
 import path from "path";
 
-dotenv.config();
-
 // Initialize directories
 const tweetsDir = path.join(process.cwd(), 'data', 'tweets');
 if (!fs.existsSync(tweetsDir)) {
     fs.mkdirSync(tweetsDir, { recursive: true });
 }
 
-async function initTwitterScraper() {
-    const scraper = new Scraper();
-    let isAuthenticated = false;
-
-    // Check for existing cookies
-    if (fs.existsSync('./cookies.json')) {
-        try {
-            const cookiesText = fs.readFileSync('./cookies.json', 'utf8');
-            const cookiesArray = JSON.parse(cookiesText);
-
-            // Format cookies for setting
-            const cookieStrings = cookiesArray.map(cookie => 
-                `${cookie.key}=${cookie.value}; Domain=${cookie.domain}; Path=${cookie.path}; ` +
-                `${cookie.secure ? 'Secure' : ''}; ${cookie.httpOnly ? 'HttpOnly' : ''}; ` +
-                `SameSite=${cookie.sameSite || 'Lax'}`
-            );
-
-            await scraper.setCookies(cookieStrings);
-            isAuthenticated = await scraper.isLoggedIn();
-            console.log('Loaded existing cookies:', isAuthenticated ? 'success' : 'failed');
-        } catch (e) {
-            console.error('Error loading cookies:', e);
-        }
-    }
-
-    // If no valid cookies, login with credentials
-    if (!isAuthenticated) {
-        if (!process.env.TWITTER_USERNAME || !process.env.TWITTER_PASSWORD) {
-            throw new Error('Twitter credentials are required');
-        }
-
-        try {
-            await scraper.login(
-                process.env.TWITTER_USERNAME,
-                process.env.TWITTER_PASSWORD,
-                process.env.TWITTER_EMAIL
-            );
-
-            // Save cookies for future use
-            const cookies = await scraper.getCookies();
-            fs.writeFileSync('./cookies.json', JSON.stringify(cookies, null, 2));
-            console.log('Logged in and saved new cookies');
-        } catch (e) {
-            console.error('Login failed:', e);
-            throw e;
-        }
-    }
-
-    return scraper;
-}
-
 async function processTwitterData() {
     try {
-        const scraper = await initTwitterScraper();
-        
-        if (!(await scraper.isLoggedIn())) {
-            console.error('Failed to authenticate with Twitter');
-            process.exit(1);
-        }
-
         // Get the most recent tweets file
         const files = fs.readdirSync(tweetsDir)
             .filter(f => f.endsWith('.json') && !f.endsWith('-export.json'))
@@ -113,16 +51,8 @@ async function processTwitterData() {
 
         console.log(`Extracted ${tweetTexts.length} tweets and saved to ${outputFile}`);
 
-        // Cleanup
-        await scraper.logout();
-        console.log("Logged out successfully!");
-
     } catch (error) {
         console.error("An error occurred:", error);
-        if (error.response) {
-            console.error("Response data:", error.response.data);
-            console.error("Response status:", error.response.status);
-        }
         process.exit(1);
     }
 }

data/discord/rules/temple_page_3.csv

@@ -5,14 +5,20 @@ import path from "path";
 
 dotenv.config();
 
+// Get username from command line argument
+const TARGET_USERNAME = process.argv[2];
+if (!TARGET_USERNAME) {
+    console.error("Please provide a Twitter username as an argument");
+    console.error("Usage: ts-node twitter-scraper.ts <username>");
+    process.exit(1);
+}
+
 // Create necessary directories
 const DATA_DIR = path.join(process.cwd(), 'data', 'tweets');
 if (!fs.existsSync(DATA_DIR)) {
     fs.mkdirSync(DATA_DIR, { recursive: true });
 }
 
-const TARGET_USERNAME = "michaelmjfm";  // Change this to the user you want to scrape
-
 // Create timestamp for filename
 const timestamp = new Date().toISOString()
     .replace(/[:.]/g, '-')
@@ -21,6 +27,7 @@ const timestamp = new Date().toISOString()
 
 const TWEETS_FILE = path.join(DATA_DIR, `${TARGET_USERNAME}_${timestamp}.json`);
 
+
 async function initTwitterScraper() {
     const scraper = new Scraper();
     let isAuthenticated = false;