build-reject-domainset.ts

// @ts-check
import path from 'node:path';
import process from 'node:process';

import { processHostsWithPreload } from './lib/parse-filter/hosts';
import { processDomainListsWithPreload } from './lib/parse-filter/domainlists';
import { processFilterRulesWithPreload } from './lib/parse-filter/filters';

import { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, DOMAIN_LISTS, HOSTS_EXTRA, DOMAIN_LISTS_EXTRA, ADGUARD_FILTERS_EXTRA, ADGUARD_FILTERS_WHITELIST, PHISHING_HOSTS_EXTRA, PHISHING_DOMAIN_LISTS_EXTRA } from './constants/reject-data-source';
import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
import { task } from './trace';
// tldts-experimental is way faster than tldts, but very little bit inaccurate
// (since it is hashes based). But the result is still deterministic, which is
// enough when creating a simple stat of reject hosts.
import { SHARED_DESCRIPTION } from './constants/description';

import { addArrayElementsToSet } from 'foxts/add-array-elements-to-set';
import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir';
import { DomainsetOutput } from './lib/rules/domainset';
import { foundDebugDomain } from './lib/parse-filter/shared';
import { AdGuardHomeOutput } from './lib/rules/domainset';
import { getPhishingDomains } from './lib/get-phishing-domains';

const readLocalRejectDomainsetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject.conf'));
const readLocalRejectExtraDomainsetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject_extra.conf'));
const readLocalRejectRulesetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/reject.conf'));
const readLocalRejectDropRulesetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/reject-drop.conf'));
const readLocalRejectNoDropRulesetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/reject-no-drop.conf'));
const readLocalMyRejectRulesetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/my_reject.conf'));

const hostsDownloads = HOSTS.map(entry => processHostsWithPreload(...entry));
const hostsExtraDownloads = HOSTS_EXTRA.map(entry => processHostsWithPreload(...entry));
const domainListsDownloads = DOMAIN_LISTS.map(entry => processDomainListsWithPreload(...entry));
const domainListsExtraDownloads = DOMAIN_LISTS_EXTRA.map(entry => processDomainListsWithPreload(...entry));
const adguardFiltersDownloads = ADGUARD_FILTERS.map(entry => processFilterRulesWithPreload(...entry));
const adguardFiltersExtraDownloads = ADGUARD_FILTERS_EXTRA.map(entry => processFilterRulesWithPreload(...entry));
const adguardFiltersWhitelistsDownloads = ADGUARD_FILTERS_WHITELIST.map(entry => processFilterRulesWithPreload(...entry));

export const buildRejectDomainSet = task(require.main === module, __filename)(async (span) => {
  const rejectBaseDescription = [
    ...SHARED_DESCRIPTION,
    '',
    'The domainset supports AD blocking, tracking protection, privacy protection, anti-mining',
    '',
    'Build from:',
    ...HOSTS.map(host => ` - ${host[0]}`),
    ...DOMAIN_LISTS.map(domainList => ` - ${domainList[0]}`),
    ...ADGUARD_FILTERS.map(filter => ` - ${Array.isArray(filter) ? filter[0] : filter}`)
  ];

  const rejectOutput = new DomainsetOutput(span, 'reject')
    .withTitle('Sukka\'s Ruleset - Reject Base')
    .withDescription(rejectBaseDescription);

  const rejectExtraOutput = new DomainsetOutput(span, 'reject_extra')
    .withTitle('Sukka\'s Ruleset - Reject Extra')
    .withDescription([
      ...SHARED_DESCRIPTION,
      '',
      'The domainset supports AD blocking, tracking protection, privacy protection, anti-mining',
      '',
      'Build from:',
      ...HOSTS_EXTRA.map(host => ` - ${host[0]}`),
      ...DOMAIN_LISTS_EXTRA.map(domainList => ` - ${domainList[0]}`),
      ...ADGUARD_FILTERS_EXTRA.map(filter => ` - ${filter[0]}`)
    ]);

  const rejectPhisingOutput = new DomainsetOutput(span, 'reject_phishing')
    .withTitle('Sukka\'s Ruleset - Reject Phishing')
    .withDescription([
      ...SHARED_DESCRIPTION,
      '',
      'The domainset is specifically designed for anti-phishing',
      '',
      'Build from:',
      ...PHISHING_HOSTS_EXTRA.map(host => ` - ${host[0]}`),
      ...PHISHING_DOMAIN_LISTS_EXTRA.map(domainList => ` - ${domainList[0]}`)
    ]);

  const appendArrayToRejectOutput = rejectOutput.addFromDomainset.bind(rejectOutput);
  const appendArrayToRejectExtraOutput = rejectExtraOutput.addFromDomainset.bind(rejectExtraOutput);

  /** Whitelists */
  const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);

  // Parse from AdGuard Filters
  await span
    .traceChild('download and process hosts / adblock filter rules')
    .traceAsyncFn((childSpan) => Promise.all([
      // Dedupe domainSets
      // Collect DOMAIN, DOMAIN-SUFFIX, and DOMAIN-KEYWORD from non_ip/reject.conf for deduplication
      // DOMAIN-WILDCARD is not really useful for deduplication, it is only included in AdGuardHome output
      // It is faster to add base than add others first then whitelist
      rejectOutput.addFromRuleset(readLocalRejectRulesetPromise),
      rejectExtraOutput.addFromRuleset(readLocalRejectRulesetPromise),

      rejectOutput.addFromDomainset(readLocalRejectDomainsetPromise),
      rejectExtraOutput.addFromDomainset(readLocalRejectDomainsetPromise),
      rejectPhisingOutput.addFromDomainset(readLocalRejectDomainsetPromise),

      rejectExtraOutput.addFromDomainset(readLocalRejectExtraDomainsetPromise),

      // Parse from remote hosts & domain lists
      hostsDownloads.map(task => task(childSpan).then(appendArrayToRejectOutput)),
      hostsExtraDownloads.map(task => task(childSpan).then(appendArrayToRejectExtraOutput)),

      domainListsDownloads.map(task => task(childSpan).then(appendArrayToRejectOutput)),
      domainListsExtraDownloads.map(task => task(childSpan).then(appendArrayToRejectExtraOutput)),

      rejectPhisingOutput.addFromDomainset(getPhishingDomains(childSpan)),

      adguardFiltersDownloads.map(
        task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes }) => {
          addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomains);
          addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomainSuffixes, suffix => '.' + suffix);

          rejectOutput.bulkAddDomain(blackDomains);
          rejectOutput.bulkAddDomainSuffix(blackDomainSuffixes);
        })
      ),
      adguardFiltersExtraDownloads.map(
        task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes }) => {
          addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomains);
          addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomainSuffixes, suffix => '.' + suffix);

          rejectExtraOutput.bulkAddDomain(blackDomains);
          rejectExtraOutput.bulkAddDomainSuffix(blackDomainSuffixes);
        })
      ),
      adguardFiltersWhitelistsDownloads.map(
        task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes }) => {
          addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomains);
          addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomainSuffixes, suffix => '.' + suffix);
          addArrayElementsToSet(filterRuleWhitelistDomainSets, blackDomains);
          addArrayElementsToSet(filterRuleWhitelistDomainSets, blackDomainSuffixes, suffix => '.' + suffix);
        })
      )
    ].flat()));

  if (foundDebugDomain.value) {
    // eslint-disable-next-line sukka/unicorn/no-process-exit -- cli App
    process.exit(1);
  }

  await Promise.all([
    rejectOutput.done(),
    rejectExtraOutput.done(),
    rejectPhisingOutput.done()
  ]);

  // whitelist
  span.traceChildSync('whitelist', () => {
    for (const domain of filterRuleWhitelistDomainSets) {
      rejectOutput.whitelistDomain(domain);
      rejectExtraOutput.whitelistDomain(domain);
      rejectPhisingOutput.whitelistDomain(domain);
    }

    rejectOutput.domainTrie.dump(rejectExtraOutput.whitelistDomain.bind(rejectExtraOutput));
    rejectOutput.domainTrie.dump(rejectPhisingOutput.whitelistDomain.bind(rejectPhisingOutput));
  });

  await Promise.all([
    rejectOutput.write(),
    rejectExtraOutput.write(),
    rejectPhisingOutput.write()
  ]);

  // we are going to re-use rejectOutput's domainTrie and mutate it
  // so we must wait until we write rejectOutput to disk after we can mutate its trie
  const rejectOutputAdGuardHome = new AdGuardHomeOutput(span, 'reject-adguardhome', OUTPUT_INTERNAL_DIR)
    .withTitle('Sukka\'s Ruleset - Blocklist for AdGuardHome')
    .withDescription([
      'The AdGuardHome ruleset supports AD blocking, tracking protection, privacy protection, anti-mining'
    ]);

  rejectOutputAdGuardHome.domainTrie = rejectOutput.domainTrie;

  await rejectOutputAdGuardHome
    .addFromRuleset(readLocalMyRejectRulesetPromise)
    .addFromRuleset(readLocalRejectRulesetPromise)
    .addFromRuleset(readLocalRejectDropRulesetPromise)
    .addFromRuleset(readLocalRejectNoDropRulesetPromise)
    .addFromDomainset(readLocalRejectExtraDomainsetPromise)
    // .
    .write();
});