add full metadata switching, more vpn ips

package.json

@@ -17,6 +17,7 @@
     "@elysiajs/static": "^1.4.0",
     "@types/html-minifier-next": "^2.1.0",
     "@types/pg": "^8.11.10",
+    "@types/ws": "^8.18.1",
     "age-encryption": "^0.2.4",
     "date-fns": "^4.1.0",
     "elysia": "^1.1.25",

ranges/expressvpn.txt

@@ -6,6 +6,7 @@
 146.19.7.128/25
 62.169.132.0/24
 14.102.85.0/24
+158.173.167.0/24
 2a0f:e385::/32
 14.102.52.0/24
 155.2.180.0/24
@@ -34,22 +35,27 @@
 14.102.53.0/24
 185.64.78.0/25
 185.64.78.128/25
+158.173.65.0/24
 103.213.214.0/24
 103.61.198.0/24
 157.97.122.0/24
+158.173.67.0/24
 185.114.34.0/24
 198.55.29.0/24
 2a0e:4203::/32
 155.2.217.0/24
 170.62.235.0/24
 170.62.244.0/24
+158.173.240.0/24
 170.62.245.0/24
+158.173.242.0/24
 170.62.237.0/24
 66.56.86.0/24
 158.173.36.0/24
 62.169.134.0/24
 157.97.120.0/24
 158.173.49.0/24
+158.173.74.0/24
 170.62.246.0/24
 198.55.30.0/24
 2a0b:64c3::/32
@@ -152,6 +158,7 @@
 81.95.58.0/24
 81.95.59.0/24
 14.102.87.0/24
+158.173.165.0/24
 45.130.81.0/24
 107.150.166.0/24
 136.144.33.0/24
@@ -178,6 +185,8 @@
 2a11:53c5::/32
 158.173.4.0/24
 158.173.5.0/24
+158.173.162.0/24
+158.173.244.0/24
 185.245.5.0/24
 203.188.167.0/24
 212.78.246.0/24
@@ -213,16 +222,20 @@
 158.173.135.0/24
 212.32.50.0/24
 45.130.141.0/24
+158.173.72.0/24
 188.240.73.0/24
 212.56.49.0/24
 45.91.23.0/24
+158.173.164.0/24
 212.32.72.0/24
 2a0f:f42::/32
 45.154.138.0/24
+158.173.77.0/24
 170.62.238.0/24
 45.91.20.0/24
 2a07:e342::/32
 194.61.40.0/24
+158.173.243.0/24
 194.61.41.0/24
 62.169.128.0/24
 217.119.143.0/25
@@ -302,11 +315,13 @@
 203.25.124.0/25
 203.25.124.128/25
 212.32.76.0/24
+158.173.166.0/24
 170.62.247.0/24
 212.78.244.0/24
 45.150.93.0/24
 45.92.228.0/24
 2a0b:64c2::/32
+158.173.66.0/24
 170.62.228.0/24
 158.173.157.0/24
 158.173.158.0/24
@@ -391,6 +406,7 @@
 89.47.15.0/25
 89.47.15.128/25
 2a0e:4205::/32
+158.173.241.0/24
 158.173.44.0/24
 170.62.226.0/24
 170.62.89.0/24
@@ -418,6 +434,7 @@
 193.39.215.128/25
 85.8.130.0/25
 85.8.130.128/25
+158.173.76.0/24
 203.188.169.0/24
 2a0b:64c6::/32
 103.61.199.0/24
@@ -439,6 +456,7 @@
 89.251.0.0/24
 93.115.255.0/24
 2a0f:e386::/32
+158.173.75.0/24
 45.86.200.0/24
 85.203.13.0/24
 155.2.178.0/24
@@ -455,6 +473,7 @@
 2a0e:4202::/32
 158.173.16.0/24
 158.173.17.0/24
+158.173.78.0/24
 192.253.210.0/24
 188.213.202.0/24
 194.5.49.0/24
@@ -467,8 +486,10 @@
 45.148.25.0/24
 45.95.243.0/24
 2a0e:d783::/32
+158.173.73.0/24
 212.56.52.0/24
 62.169.135.0/24
+158.173.79.0/24
 136.144.42.0/24
 136.144.27.0/24
 170.62.230.0/24

ranges/urbanvpn.txt

@@ -47,13 +47,11 @@
 67.43.236.226/32
 148.113.221.150/32
 148.113.221.152/32
-173.209.51.250/32
 104.245.146.82/32
 135.84.180.228/32
 173.209.48.162/32
 173.209.49.50/32
 179.43.152.90/32
-146.70.135.14/32
 208.69.78.7/32
 66.163.116.199/32
 102.220.17.118/32
@@ -70,17 +68,17 @@
 217.138.220.50/32
 217.138.220.226/32
 217.138.220.94/32
-89.163.221.18/32
+213.202.254.242/32
-193.108.116.248/32
+193.108.116.242/32
-193.108.116.232/32
+193.108.116.226/32
-23.160.72.45/32
+23.160.72.37/32
-23.160.72.205/32
+23.160.72.206/32
-23.160.72.218/32
+23.160.72.211/32
-23.158.56.244/32
+193.108.116.218/32
-23.160.72.50/32
+23.160.72.56/32
-51.38.111.185/32
+57.129.88.70/32
-51.38.121.218/32
+51.38.121.161/32
-85.114.138.43/32
+5.104.107.68/32
 5.104.107.251/32
 82.103.131.250/32
 146.70.42.202/32
@@ -134,6 +132,7 @@
 95.129.46.100/32
 170.80.111.119/32
 169.150.222.197/32
+61.4.121.186/32
 190.92.9.46/32
 178.218.162.117/32
 185.104.187.130/32
@@ -147,7 +146,6 @@
 185.253.73.216/32
 182.54.236.193/32
 182.54.236.194/32
-148.113.8.93/32
 148.113.0.105/32
 148.113.47.86/32
 148.113.0.104/32
@@ -170,7 +168,6 @@
 102.68.86.97/32
 91.213.233.111/32
 91.213.233.176/32
-79.110.55.34/32
 61.255.174.11/32
 140.174.179.129/32
 38.54.124.170/32
@@ -197,7 +194,6 @@
 131.196.35.40/32
 167.17.70.173/32
 172.99.188.95/32
-51.15.16.66/32
 185.181.61.141/32
 83.143.82.62/32
 83.143.82.58/32
@@ -218,7 +214,6 @@
 199.255.116.5/32
 38.158.220.26/32
 185.113.141.65/32
-185.113.140.26/32
 185.113.140.45/32
 38.165.233.7/32
 38.165.233.30/32
@@ -252,13 +247,14 @@
 176.103.50.127/32
 176.103.54.71/32
 146.70.228.82/32
-167.17.66.82/32
 169.197.83.34/32
 169.197.85.171/32
+162.249.172.18/32
 169.197.85.170/32
 169.197.142.208/32
 38.128.66.22/32
 38.68.134.126/32
+169.197.142.119/32
 162.251.62.66/32
 23.154.136.106/32
 169.197.85.172/32

ranges/vpnly.txt

@@ -51,3 +51,44 @@
 108.181.58.33/32
 46.229.243.203/32
 46.229.243.178/32
+80.92.204.6/32
+80.92.204.74/32
+45.12.138.188/32
+80.71.157.209/32
+80.92.204.84/32
+45.83.129.161/32
+80.92.204.82/32
+45.14.247.113/32
+80.92.204.73/32
+80.92.204.75/32
+80.71.157.218/32
+80.71.157.239/32
+80.92.204.85/32
+80.92.204.57/32
+80.92.204.42/32
+80.92.204.62/32
+208.87.240.255/32
+108.181.0.171/32
+108.181.0.21/32
+108.181.3.145/32
+208.87.241.235/32
+208.87.241.221/32
+108.181.1.241/32
+208.87.242.111/32
+208.87.242.23/32
+108.181.4.69/32
+208.87.240.19/32
+108.181.3.149/32
+108.181.0.109/32
+208.87.242.199/32
+45.12.133.18/32
+45.12.133.12/32
+94.232.247.207/32
+94.232.247.205/32
+45.12.133.22/32
+46.229.243.200/32
+46.229.243.201/32
+46.229.243.203/32
+46.229.243.197/32
+46.229.243.199/32
+46.229.243.178/32

scripts/1clickvpn.ts

@@ -0,0 +1,2 @@
+const oneClickVpn = await (await fetch('https://1clickvpn.net/api/v1/servers/')).json()
+Bun.write('ranges/1clickvpn.txt', oneClickVpn.flatMap(v => v.nodes.map(n => n.ip + '/32')).join('\n'))

scripts/expressvpn.ts

@@ -0,0 +1,54 @@
+import { sleep } from "bun";
+
+const file = Bun.file('ranges/expressvpn.txt');
+const writer = file.writer();
+
+function inetnumToCIDR(value: string): string[] {
+  let [start, end] = value.split(' - ').map(ip =>
+    ip.split('.').reduce((acc, octet) => (acc << 8) | parseInt(octet), 0) >>> 0
+  )
+
+  const cidrs: string[] = []
+
+  while (start <= end) {
+    const maxBits = Math.floor(Math.log2(start & -start)) // largest block start allows
+    const fitBits = Math.floor(Math.log2(end - start + 1)) // largest block size fits
+    const size = Math.min(maxBits, fitBits)
+    const ip = [(start >>> 24) & 255, (start >>> 16) & 255, (start >>> 8) & 255, start & 255].join('.')
+    cidrs.push(`${ip}/${32 - size}`)
+    start += 2 ** size
+  }
+
+  return cidrs
+}
+
+const orgs = await (await fetch('https://apps.db.ripe.net/db-web-ui/api/whois/search?abuse-contact=true&ignore404=true&managed-attributes=true&resource-holder=true&type-filter=ORGANISATION&flags=r&offset=0&limit=200&query-string=VPN%20Consumer')).json()
+const orgId = orgs.objects.object.map(o => o.attributes.attribute.find(a => a.name === 'organisation').value)
+
+for (const o of orgId) {
+  const orgReq = await fetch(`https://apps.db.ripe.net/db-web-ui/api/whois/search?abuse-contact=true&ignore404=true&managed-attributes=true&resource-holder=true&type-filter=INETNUM,INET6NUM&inverse-attribute=ORG&flags=r&offset=0&limit=200&query-string=${o}`)
+  if (orgReq.status == 404) {
+    console.log(`no inetnum/inet6num for ${o}`)
+    continue
+  }
+  const org = await orgReq.json()
+  const primaries = org.objects.object.map(o => o['primary-key'].attribute[0])
+  for (const p of primaries) {
+    if (p.name == 'inetnum') {
+      console.log(inetnumToCIDR(p.value).join('\n'))
+      writer.write(inetnumToCIDR(p.value).join('\n') + '\n')
+    } else if (p.name == 'inet6num') {
+      console.log(p.value)
+      writer.write(p.value + '\n')
+    }
+  }
+
+  writer.flush()
+
+  if (orgReq.headers.get('X-Rate-Limit-Remaining') == '1') {
+    console.log('sleeping 5s')
+    await sleep(5000)
+  }
+}
+
+writer.end();

scripts/urbanvpn.ts

@@ -0,0 +1,22 @@
+const post = await (await fetch('https://api-pro.falais.com/rest/v1/security/tokens/accs', {
+  method: 'POST',
+  headers: {
+    'authorization': 'Bearer FihZXBoQi83OomWPQgj9VqEFPzRsLz6p',
+    'content-type': 'application/json'
+  },
+  body: JSON.stringify({
+    "type": "accs",
+    "clientApp": {
+      "name": "URBAN_VPN_BROWSER_EXTENSION"
+    }
+  })
+})).json()
+
+const servers = await (await fetch('https://stats.falais.com/api/rest/v2/entrypoints/countries', {
+  headers: {
+    'authorization': `Bearer ${post.value}`,
+    'x-client-app': 'URBAN_VPN_BROWSER_EXTENSION'
+  }
+})).json()
+
+Bun.write('ranges/urbanvpn.txt', servers.countries.elements.flatMap(c => c.servers.elements.map(s => s.address.primary.ip + '/32')).join('\n'))

scripts/vpnly.ts

@@ -0,0 +1,32 @@
+import * as fs from 'node:fs'
+
+async function getARecords(hostname: string): Promise<string[]> {
+  const url = `https://cloudflare-dns.com/dns-query?name=${encodeURIComponent(
+    hostname
+  )}&type=A`;
+
+  const res = await fetch(url, {
+    headers: { Accept: "application/dns-json" },
+  });
+
+  if (!res.ok) {
+    throw new Error(`dns query failed: ${res.status} ${res.statusText}`);
+  }
+
+  const body = (await res.json()) as {
+    Status: number;
+    Answer?: Array<{ name: string; type: number; data: string }>;
+  };
+
+  if (body.Status !== 0 || !body.Answer) {
+    return [];
+  }
+
+  return body.Answer.filter((a) => a.type === 1).map((a) => a.data);
+}
+
+const hostnames = ['de-hub.freeruproxy.ink', 'us-hub.freeruproxy.ink', 'fr-hub.freeruproxy.ink', 'nl-hub.freeruproxy.ink']
+for (const h of hostnames) {
+  const records = await getARecords(h)
+  fs.appendFileSync('ranges/vpnly.txt', records.map(r => r + '/32').join('\n') + '\n')
+}

src/utils/health.ts

@@ -1,5 +1,13 @@
 const healthStatus: Record<string, string> = {}
 
+function getMetadataBackend() {
+  const primary = process.env.METADATA!
+  const alternative = process.env.ALTERNATIVE_METADATA!
+
+  if (healthStatus[primary] === 'healthy' || !alternative) return primary
+  return alternative
+}
+
 async function checkHealth() {
   const metadataServers: string[] = [process.env.METADATA!, process.env.ALTERNATIVE_METADATA!]
   await Promise.all(metadataServers.map(async (m) => {
@@ -18,3 +26,4 @@ checkHealth()
 setInterval(checkHealth, 5 * 60000)
 
 export default healthStatus
+export { getMetadataBackend }

src/utils/metadata.ts

@@ -1,15 +1,16 @@
 // metadata either returns innertube or { error: string }
+import { getMetadataBackend } from '@/utils/health';
 
 async function getVideo(id: string) {
-  return await (await fetch(`${process.env.METADATA}/video/${id}`)).json()
+  return await (await fetch(`${getMetadataBackend()}/video/${id}`)).json()
 }
 
 async function getChannel(id: string) {
-  return await (await fetch(`${process.env.METADATA}/channel/${id}`)).json()
+  return await (await fetch(`${getMetadataBackend()}/channel/${id}`)).json()
 }
 
 async function getChannelVideos(id: string) {
-  return await (await fetch(`${process.env.METADATA}/videos/${id}`)).json()
+  return await (await fetch(`${getMetadataBackend()}/videos/${id}`)).json()
 }
 
 export { getVideo, getChannel, getChannelVideos }

src/utils/regex.ts

@@ -1,3 +1,5 @@
+import { getMetadataBackend } from '@/utils/health';
+
 function validateVideo(input: string): string | false {
   try {
     const url = new URL(input);
@@ -87,7 +89,7 @@ async function validateChannel(input: string): Promise<string | { error: string;
         whatIsIt = 'handle'
       } else return false 
 
-      const channelReq = await fetch(`${process.env.METADATA}/getWebpageJson?url=${url}`)
+      const channelReq = await fetch(`${getMetadataBackend()}/getWebpageJson?url=${url}`)
       if (!channelReq.ok) return {
         error: `Failed to fetch Youtube with status ${channelReq.status}. Please retry.`
       }