fix(test): after rebase + added referrars test and more sites

Author: lindesvard

apps/api/package.json

@@ -6,7 +6,6 @@
     "testing": "API_PORT=3333 pnpm dev",
     "start": "node dist/index.js",
     "build": "rm -rf dist && tsup",
-    "gen:referrers": "jiti scripts/get-referrers.ts && biome format --write src/referrers/index.ts",
     "gen:bots": "jiti scripts/get-bots.ts && biome format --write src/bots/bots.ts",
     "typecheck": "tsc --noEmit"
   },

apps/api/scripts/get-referrers.ts

@@ -7,7 +7,8 @@
     "testing": "WORKER_PORT=9999 pnpm dev",
     "start": "node dist/index.js",
     "build": "rm -rf dist && tsup",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc --noEmit",
+    "gen:referrers": "jiti scripts/get-referrers.ts && biome format --write ./src/referrers/index.ts"
   },
   "dependencies": {
     "@bull-board/api": "5.21.0",

apps/api/src/referrers/index.ts

@@ -0,0 +1,91 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+// extras
+const extraReferrers = {
+  'zoom.us': { type: 'social', name: 'Zoom' },
+  'apple.com': { type: 'tech', name: 'Apple' },
+  'adobe.com': { type: 'tech', name: 'Adobe' },
+  'figma.com': { type: 'tech', name: 'Figma' },
+  'wix.com': { type: 'commerce', name: 'Wix' },
+  'gmail.com': { type: 'email', name: 'Gmail' },
+  'notion.so': { type: 'tech', name: 'Notion' },
+  'ebay.com': { type: 'commerce', name: 'eBay' },
+  'github.com': { type: 'tech', name: 'GitHub' },
+  'gitlab.com': { type: 'tech', name: 'GitLab' },
+  'slack.com': { type: 'social', name: 'Slack' },
+  'etsy.com': { type: 'commerce', name: 'Etsy' },
+  'bsky.app': { type: 'social', name: 'Bluesky' },
+  'twitch.tv': { type: 'content', name: 'Twitch' },
+  'dropbox.com': { type: 'tech', name: 'Dropbox' },
+  'outlook.com': { type: 'email', name: 'Outlook' },
+  'medium.com': { type: 'content', name: 'Medium' },
+  'paypal.com': { type: 'commerce', name: 'PayPal' },
+  'discord.com': { type: 'social', name: 'Discord' },
+  'stripe.com': { type: 'commerce', name: 'Stripe' },
+  'spotify.com': { type: 'content', name: 'Spotify' },
+  'netflix.com': { type: 'content', name: 'Netflix' },
+  'whatsapp.com': { type: 'social', name: 'WhatsApp' },
+  'shopify.com': { type: 'commerce', name: 'Shopify' },
+  'microsoft.com': { type: 'tech', name: 'Microsoft' },
+  'alibaba.com': { type: 'commerce', name: 'Alibaba' },
+  'telegram.org': { type: 'social', name: 'Telegram' },
+  'substack.com': { type: 'content', name: 'Substack' },
+  'salesforce.com': { type: 'tech', name: 'Salesforce' },
+  'instagram.com': { type: 'social', name: 'Instagram' },
+  'wikipedia.org': { type: 'content', name: 'Wikipedia' },
+  'mastodon.social': { type: 'social', name: 'Mastodon' },
+  'office.com': { type: 'tech', name: 'Microsoft Office' },
+  'squarespace.com': { type: 'commerce', name: 'Squarespace' },
+  'stackoverflow.com': { type: 'tech', name: 'Stack Overflow' },
+  'teams.microsoft.com': { type: 'social', name: 'Microsoft Teams' },
+};
+
+function transform(data: any) {
+  const obj: Record<string, unknown> = {};
+  for (const type in data) {
+    for (const name in data[type]) {
+      const domains = data[type][name].domains ?? [];
+      for (const domain of domains) {
+        obj[domain] = {
+          type,
+          name,
+        };
+      }
+    }
+  }
+
+  return obj;
+}
+
+async function main() {
+  // Get document, or throw exception on error
+  try {
+    const data = await fetch(
+      'https://s3-eu-west-1.amazonaws.com/snowplow-hosted-assets/third-party/referer-parser/referers-latest.json',
+    ).then((res) => res.json());
+
+    fs.writeFileSync(
+      path.resolve(__dirname, '../../worker/src/referrers/index.ts'),
+      [
+        '// This file is generated by the script get-referrers.ts',
+        '',
+        '// The data is fetch from snowplow-referer-parser https://github.com/snowplow-referer-parser/referer-parser',
+        `// The orginal referers.yml is based on Piwik's SearchEngines.php and Socials.php, copyright 2012 Matthieu Aubry and available under the GNU General Public License v3.`,
+        '',
+        `const referrers: Record<string, { type: string, name: string }> = ${JSON.stringify(
+          {
+            ...transform(data),
+            ...extraReferrers,
+          },
+        )} as const;`,
+        'export default referrers;',
+      ].join('\n'),
+      'utf-8',
+    );
+  } catch (e) {
+    console.log(e);
+  }
+}
+
+main();