Indexing posts from Jetstream (#13)

* Wrote a client to consume incoming data from a Jetstream instance rather than the full firehose.

* Only posts are consumed in the Jetstream client.

* Registered users are now periodically synced rather than calculated from incoming follow and unfollow events.

* Patch for set cursor to 0.0 on launch.

---------

Co-authored-by: Joel Hill <joel.hill@solusguard.com>

Author: joelghill

.vscode/launch.json

@@ -4,28 +4,14 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+
         {
-            "name": "Python: Log Feed",
-            "type": "debugpy",
-            "request": "launch",
-            "program": "${workspaceFolder}/manage.py",
-            "args": [
-                "start_feed",
-                "wss://bsky.network/xrpc",
-                "--algorithm=logger"
-            ],
-            "django": true,
-            "justMyCode": false
-        },
-        {
-            "name": "Python: Index Feed",
+            "name": "Python: Index Posts",
             "type": "debugpy",
             "request": "launch",
             "program": "${workspaceFolder}/manage.py",
             "args": [
                 "start_feed",
-                "wss://bsky.network/xrpc",
-                "--algorithm=flatlanders"
             ],
             "django": true,
             "justMyCode": false
@@ -42,7 +28,7 @@
             "justMyCode": false
         },
         {
-            "name": "Python: labeler",
+            "name": "Python: Labeler",
             "type": "debugpy",
             "request": "launch",
             "program": "${workspaceFolder}/manage.py",
@@ -53,6 +39,5 @@
             "django": true,
             "justMyCode": false
         }
-
     ]
 }

.vscode/settings.json

@@ -3,5 +3,16 @@
         "tests"
     ],
     "python.testing.unittestEnabled": false,
-    "python.testing.pytestEnabled": true
+    "python.testing.pytestEnabled": true,
+    "cSpell.words": [
+        "afrom",
+        "atproto",
+        "bsky",
+        "dids",
+        "FEEDGEN",
+        "Flatlander",
+        "flatlanders",
+        "Jetstream",
+        "xrpc"
+    ]
 }

Pipfile

@@ -12,6 +12,8 @@ atproto = "*"
 python-dotenv = "*"
 sentry-sdk = {extras = ["django"], version = "*"}
 uvloop = "*"
+websockets = "*"
+zstandard = "*"
 
 [dev-packages]
 djlint = "*"

Pipfile.lock

@@ -4,42 +4,32 @@ set -e
 
 start_server() {
     echo "Starting server..."
-    /app/.venv/bin/python manage.py migrate
     exec /app/.venv/bin/gunicorn -c gunicorn.conf.py sk_atp_feed.wsgi
 }
 
 start_indexer() {
-    echo "Starting indexer..."
-    exec /app/.venv/bin/python manage.py start_feed "wss://bsky.network/xrpc" --algorithm=flatlanders
+    echo "Starting posts indexer..."
+    /app/.venv/bin/python manage.py migrate
+    exec /app/.venv/bin/python manage.py start_feed
 }
 
 start_labeler() {
     echo "Starting labeler..."
-    /app/.venv/bin/python manage.py migrate
     exec /app/.venv/bin/python manage.py start_labeler
 }
 
-start_nginx() {
-    echo "Starting server..."
-    /app/.venv/bin/python manage.py migrate
-    exec nginx -g "daemon off;"
-}
-
 case $1 in
 server)
     start_server
     ;;
-server)
-    start_nginx
-    ;;
 indexer)
     start_indexer
     ;;
 labeler)
     start_labeler
     ;;
 *)
-    echo "Usage: $0 {server|indexer|labeler|nginx}"
+    echo "Usage: $0 {server|indexer|labeler}"
     exit 1
     ;;
 esac