From ffca3f4bbe9c671e66771c8b72cc0c05af3e25d7 Mon Sep 17 00:00:00 2001 From: localhost Date: Thu, 2 Apr 2026 18:29:16 +0200 Subject: [PATCH] add asn blacklists --- ranges/urbanvpn.txt | 44 +++++++---- src/utils/ranges.ts | 182 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 199 insertions(+), 27 deletions(-) diff --git a/ranges/urbanvpn.txt b/ranges/urbanvpn.txt index d7215ef..1ba1268 100644 --- a/ranges/urbanvpn.txt +++ b/ranges/urbanvpn.txt @@ -41,17 +41,18 @@ 140.174.187.15/32 108.181.68.87/32 140.174.187.17/32 -178.172.217.19/32 178.172.217.13/32 173.209.63.146/32 67.43.236.226/32 148.113.221.150/32 148.113.221.152/32 +173.209.51.250/32 104.245.146.82/32 135.84.180.228/32 173.209.48.162/32 173.209.49.50/32 179.43.152.90/32 +146.70.135.14/32 208.69.78.7/32 66.163.116.199/32 102.220.17.118/32 @@ -62,6 +63,7 @@ 43.252.167.96/32 103.219.169.75/32 138.121.203.146/32 +138.59.135.94/32 200.122.181.11/32 157.97.132.199/32 185.191.206.100/32 @@ -69,16 +71,16 @@ 217.138.220.226/32 217.138.220.94/32 213.202.254.242/32 -193.108.116.242/32 -193.108.116.226/32 -23.160.72.37/32 -23.160.72.206/32 -23.160.72.211/32 -193.108.116.218/32 -23.160.72.56/32 -57.129.88.70/32 -51.38.121.161/32 -5.104.107.68/32 +193.108.118.74/32 +193.108.116.237/32 +193.108.118.150/32 +193.108.116.72/32 +193.108.117.30/32 +193.108.116.213/32 +185.177.229.92/32 +51.38.111.185/32 +51.38.121.218/32 +85.114.138.43/32 5.104.107.251/32 82.103.131.250/32 146.70.42.202/32 @@ -129,12 +131,14 @@ 188.93.91.2/32 188.93.90.66/32 169.255.56.147/32 +93.174.121.131/32 95.129.46.100/32 170.80.111.119/32 169.150.222.197/32 61.4.121.186/32 190.92.9.46/32 178.218.162.117/32 +178.218.162.114/32 185.104.187.130/32 185.252.223.226/32 185.252.223.210/32 @@ -163,15 +167,16 @@ 180.149.230.169/32 103.108.230.31/32 103.108.230.51/32 -5.180.44.194/32 180.149.230.241/32 102.68.86.97/32 91.213.233.111/32 91.213.233.176/32 +79.110.55.34/32 61.255.174.11/32 140.174.179.129/32 38.54.124.170/32 185.120.77.104/32 +202.124.164.76/32 185.64.104.88/32 93.115.25.31/32 23.109.137.14/32 @@ -180,8 +185,12 @@ 23.109.136.54/32 23.109.136.58/32 23.109.136.146/32 +84.38.134.139/32 109.248.149.173/32 154.70.207.190/32 +154.70.207.146/32 +43.231.113.84/32 +43.231.114.178/32 171.22.254.19/32 171.22.254.144/32 201.150.33.188/32 @@ -194,6 +203,7 @@ 131.196.35.40/32 167.17.70.173/32 172.99.188.95/32 +51.15.16.66/32 185.181.61.141/32 83.143.82.62/32 83.143.82.58/32 @@ -204,6 +214,7 @@ 180.149.231.71/32 200.74.244.7/32 190.97.163.17/32 +190.120.229.196/32 138.186.143.50/32 103.103.0.21/32 112.199.95.186/32 @@ -214,6 +225,7 @@ 199.255.116.5/32 38.158.220.26/32 185.113.141.65/32 +185.113.140.26/32 185.113.140.45/32 38.165.233.7/32 38.165.233.30/32 @@ -233,10 +245,10 @@ 103.253.27.6/32 195.80.150.194/32 195.80.150.202/32 +165.231.211.122/32 193.37.255.2/32 185.245.85.126/32 119.59.98.133/32 -119.59.98.74/32 188.213.34.178/32 185.169.64.46/32 188.213.34.126/32 @@ -247,12 +259,13 @@ 176.103.50.127/32 176.103.54.71/32 146.70.228.82/32 +167.17.66.82/32 +130.195.212.66/32 169.197.83.34/32 169.197.85.171/32 162.249.172.18/32 169.197.85.170/32 169.197.142.208/32 -38.128.66.22/32 38.68.134.126/32 169.197.142.119/32 162.251.62.66/32 @@ -264,5 +277,4 @@ 45.228.67.28/32 103.97.125.216/32 103.9.78.107/32 -129.232.237.178/32 -129.232.237.210/32 \ No newline at end of file +129.232.237.178/32 \ No newline at end of file diff --git a/src/utils/ranges.ts b/src/utils/ranges.ts index 1f59e04..d3a4892 100644 --- a/src/utils/ranges.ts +++ b/src/utils/ranges.ts @@ -1,4 +1,4 @@ -import { readdir, readFile } from 'node:fs/promises' +import { readdir } from 'node:fs/promises' import * as path from 'node:path' export interface BlockedIpResult { @@ -9,14 +9,39 @@ export interface BlockedIpResult { const IPV6_BITS = 128n const IPV6_FULL_MASK = (1n << IPV6_BITS) - 1n +const NETWORKS_REFRESH_INTERVAL_MS = 6 * 60 * 60 * 1000 // 6h +const BLOCKED_DIR = path.resolve(process.cwd(), 'ranges') +const asnBanList: number[] = [ + 206092, // expressvpn + 63023, // free vpn servers + 14618, // useless scrapers with chromium runing + 137409, // gsl +] + type ParsedIp = | { version: 4; value: number } | { version: 6; value: bigint } type ParsedCidr = - | { version: 4; network: number; mask: number } - | { version: 6; network: bigint; mask: bigint } + | { version: 4; network: number; mask: number; prefix: number } + | { version: 6; network: bigint; mask: bigint; prefix: number } + +interface AsnMatch { + asn: number | null + range: string | null +} + +interface NetworkAsnRecord { + asn: number + cidr: string + parsed: ParsedCidr +} + +let networksCache: NetworkAsnRecord[] | null = null +let networkRefreshPromise: Promise | null = null +let networksCheckedAt = 0 +let networksEtag: string | null = null function ipv4ToInt(ip: string): number | null { const parts = ip.trim().split('.') @@ -105,7 +130,7 @@ function parseCidr(cidr: string): ParsedCidr | null { if (ip.version === 4) { if (prefix < 0 || prefix > 32) return null const mask = prefix === 0 ? 0 : (0xffffffff << (32 - prefix)) >>> 0 - return { version: 4, network: ip.value & mask, mask } + return { version: 4, network: ip.value & mask, mask, prefix } } if (prefix < 0 || prefix > 128) return null @@ -114,7 +139,7 @@ function parseCidr(cidr: string): ParsedCidr | null { ? 0n : (IPV6_FULL_MASK ^ ((1n << (IPV6_BITS - BigInt(prefix))) - 1n)) & IPV6_FULL_MASK - return { version: 6, network: ip.value & mask, mask } + return { version: 6, network: ip.value & mask, mask, prefix } } function isIpInCidr(ip: ParsedIp, cidr: string): boolean { @@ -150,14 +175,130 @@ function extractCidrs(text: string): string[] { return cidrs } +function parseCsvNetworks(text: string): NetworkAsnRecord[] { + const records: NetworkAsnRecord[] = [] + const lines = text.split(/\r?\n/) + + for (const line of lines) { + if (!line || line === 'network,asn,organization,country') continue + + const firstComma = line.indexOf(',') + const secondComma = line.indexOf(',', firstComma + 1) + if (firstComma === -1 || secondComma === -1) continue + + const cidr = line.slice(0, firstComma).trim() + const asnText = line.slice(firstComma + 1, secondComma).trim() + if (!cidr || !/^\d+$/.test(asnText)) continue + + const parsed = parseCidr(cidr) + if (!parsed) continue + + records.push({ + asn: Number(asnText), + cidr, + parsed + }) + } + + return records +} + +async function refreshNetworksCache(force: boolean = false): Promise { + if (networkRefreshPromise) return networkRefreshPromise + + networkRefreshPromise = (async () => { + const isFresh = + networksCache !== null && + networksCheckedAt > 0 && + Date.now() - networksCheckedAt < NETWORKS_REFRESH_INTERVAL_MS + + if (!force && isFresh) { + return + } + + try { + const headResponse = await fetch('https://ip.guide/bulk/networks.csv', { + method: 'HEAD', + redirect: 'follow' + }) + if (!headResponse.ok) { + throw new Error(`failed to fetch ip.guide head with ${headResponse.status}`) + } + + const remoteEtag = headResponse.headers.get('etag') + const shouldDownload = + networksCache === null || !remoteEtag || !networksEtag || remoteEtag !== networksEtag + + if (shouldDownload) { + const downloadResponse = await fetch('https://ip.guide/bulk/networks.csv', { + redirect: 'follow' + }) + if (!downloadResponse.ok) { + throw new Error(`failed to fetch ip.guide with ${downloadResponse.status}`) + } + + const content = await downloadResponse.text() + networksCache = parseCsvNetworks(content) + } + + networksCheckedAt = Date.now() + networksEtag = remoteEtag ?? networksEtag + } catch (error) { + if (networksCache === null) networksCache = [] + + console.error('Failed to refresh ASN network ranges', error) + } + })().finally(() => { + networkRefreshPromise = null + }) + + return networkRefreshPromise +} + +async function resolveIpAsn(parsedIp: ParsedIp): Promise { + await refreshNetworksCache() + const records = networksCache ?? [] + let bestMatch: NetworkAsnRecord | null = null + + for (const record of records) { + if (parsedIp.version === 4 && record.parsed.version === 4) { + if ((parsedIp.value & record.parsed.mask) === record.parsed.network) { + if (bestMatch == null || record.parsed.prefix > bestMatch.parsed.prefix) { + bestMatch = record + } + } + } + + if (parsedIp.version === 6 && record.parsed.version === 6) { + if ((parsedIp.value & record.parsed.mask) === record.parsed.network) { + if (bestMatch == null || record.parsed.prefix > bestMatch.parsed.prefix) { + bestMatch = record + } + } + } + } + + if (bestMatch) { + return { + asn: bestMatch.asn, + range: bestMatch.cidr + } + } + + return { + asn: null, + range: null + } +} + export async function checkIpRanges(ip: string): Promise { const parsedIp = parseIp(ip) if (parsedIp == null) { return { blocked: false, list: null, range: null } } - const blockedDir = path.resolve(process.cwd(), 'ranges') - const entries = await readdir(blockedDir, { withFileTypes: true }) + const asnMatch = await resolveIpAsn(parsedIp) + const entries = await readdir(BLOCKED_DIR, { withFileTypes: true }) const files = entries .filter(entry => entry.isFile() && entry.name.endsWith('.txt')) @@ -165,8 +306,8 @@ export async function checkIpRanges(ip: string): Promise { .sort((a, b) => a.localeCompare(b)) for (const fileName of files) { - const filePath = path.join(blockedDir, fileName) - const content = await readFile(filePath, 'utf8') + const filePath = path.join(BLOCKED_DIR, fileName) + const content = await Bun.file(filePath).text() const cidrs = extractCidrs(content) for (const cidr of cidrs) { @@ -180,5 +321,24 @@ export async function checkIpRanges(ip: string): Promise { } } - return { blocked: false, list: null, range: null } -} \ No newline at end of file + if (asnMatch.asn !== null && asnBanList.includes(asnMatch.asn)) { + return { + blocked: true, + list: `asn:${asnMatch.asn}`, + range: asnMatch.range + } + } + + return { + blocked: false, + list: null, + range: null + } +} + +const networkRefreshTimer = setInterval(() => { + void refreshNetworksCache(true) +}, NETWORKS_REFRESH_INTERVAL_MS) + +networkRefreshTimer.unref?.() +void refreshNetworksCache()