feat: set default headers

scripts/check-online.ts

@@ -3,7 +3,7 @@ import * as v from '@badrap/valita';
 
 import { differenceInDays } from 'date-fns/differenceInDays';
 
-import { MAX_FAILURE_DAYS } from '../src/constants';
+import { DEFAULT_HEADERS, MAX_FAILURE_DAYS } from '../src/constants';
 import { serializedState, type LabelerInfo, type PDSInfo, type SerializedState } from '../src/state';
 
 import { PromiseQueue } from '../src/utils/pqueue';
@@ -77,7 +77,7 @@ await Promise.all(
 
 			const signal = AbortSignal.timeout(15_000);
 			const meta = await rpc
-				.get('com.atproto.server.describeServer', { signal })
+				.get('com.atproto.server.describeServer', { signal, headers: DEFAULT_HEADERS })
 				.then(({ data: rawData }) => {
 					const data = pdsDescribeServerResponse.parse(rawData, { mode: 'passthrough' });
 
@@ -128,7 +128,11 @@ await Promise.all(
 
 			const signal = AbortSignal.timeout(15_000);
 			const meta = await rpc
-				.get('com.atproto.label.queryLabels', { signal: signal, params: { uriPatterns: ['*'], limit: 1 } })
+				.get('com.atproto.label.queryLabels', {
+					signal: signal,
+					headers: DEFAULT_HEADERS,
+					params: { uriPatterns: ['*'], limit: 1 },
+				})
 				.then(({ data: rawData }) => labelerQueryLabelsResponse.parse(rawData, { mode: 'passthrough' }))
 				.catch(() => null);
 
@@ -211,7 +215,7 @@ async function getVersion(rpc: XRPC, prev: string | null | undefined) {
 
 	try {
 		// @ts-expect-error: undocumented endpoint
-		const { data: rawData } = await rpc.get('_health', {});
+		const { data: rawData } = await rpc.get('_health', { headers: DEFAULT_HEADERS });
 		const { version } = offHealthResponse.parse(rawData, { mode: 'passthrough' });
 
 		return /^[0-9a-f]{40}$/.test(version) ? `git-${version.slice(0, 7)}` : version;

scripts/export-dids.ts

@@ -14,7 +14,7 @@ import {
 	type SerializedState,
 } from '../src/state';
 
-import { MAX_FAILURE_DAYS, PLC_URL, RELAY_URL } from '../src/constants';
+import { DEFAULT_HEADERS, MAX_FAILURE_DAYS, PLC_URL, RELAY_URL } from '../src/constants';
 import { didDocument, type DidDocument } from '../src/utils/did';
 import { PromiseQueue } from '../src/utils/pqueue';
 import { LineBreakStream, TextDecoderStream } from '../src/utils/stream';
@@ -177,6 +177,7 @@ let firehoseCursor: string | undefined = state?.firehose.cursor;
 
 	do {
 		const { data } = await rpc.get('com.atproto.sync.listRepos', {
+			headers: DEFAULT_HEADERS,
 			params: {
 				cursor: cursor,
 				limit: 1_000,
@@ -314,7 +315,7 @@ let firehoseCursor: string | undefined = state?.firehose.cursor;
 }
 
 async function get(url: string, signal?: AbortSignal): Promise<Response> {
-	const response = await fetch(url, { signal });
+	const response = await fetch(url, { signal, headers: DEFAULT_HEADERS });
 
 	if (response.status === 429) {
 		const delay = 90_000;

src/constants.ts

@@ -3,3 +3,8 @@ export const RELAY_URL = `https://bsky.network`;
 
 /** If `now` and `errorAt` has passed this amount of days, it should stop tracking. */
 export const MAX_FAILURE_DAYS = 14;
+
+export const USER_AGENT = 'github:mary-ext/atproto-scraping';
+export const DEFAULT_HEADERS = {
+	'user-agent': USER_AGENT,
+};