ranges update + trust based limit
This commit is contained in:
localhost
parent
2c17610353
commit
127e0fadd2
|
|
@ -20,7 +20,6 @@
|
|||
"@types/pg": "^8.11.10",
|
||||
"@types/ws": "^8.18.1",
|
||||
"age-encryption": "^0.2.4",
|
||||
"crypto-js": "^4.2.0",
|
||||
"date-fns": "^4.1.0",
|
||||
"elysia": "^1.1.25",
|
||||
"eta": "^4.0.1",
|
||||
|
|
|
|||
|
|
@ -1,10 +1,3 @@
|
|||
95.141.32.101/32
|
||||
93.115.28.181/32
|
||||
185.136.159.181/32
|
||||
79.141.162.81/32
|
||||
5.149.253.57/32
|
||||
5.149.255.178/32
|
||||
5.149.250.222/32
|
||||
77.83.198.123/32
|
||||
185.235.137.161/32
|
||||
83.143.87.194/32
|
||||
|
|
@ -5,7 +5,6 @@
|
|||
91.242.241.58/32
|
||||
193.104.75.35/32
|
||||
66.42.61.251/32
|
||||
198.145.121.235/32
|
||||
198.145.121.151/32
|
||||
91.236.230.83/32
|
||||
92.118.126.116/32
|
||||
|
|
@ -4,24 +4,30 @@
|
|||
63.125.93.136/32
|
||||
161.77.141.85/32
|
||||
176.106.62.91/32
|
||||
176.106.63.26/32
|
||||
85.255.180.8/32
|
||||
178.209.75.96/32
|
||||
83.168.84.194/32
|
||||
83.168.86.140/32
|
||||
212.68.180.226/32
|
||||
31.133.94.250/32
|
||||
31.133.95.77/32
|
||||
212.68.178.22/32
|
||||
158.46.163.29/32
|
||||
161.77.142.156/32
|
||||
208.214.161.115/32
|
||||
63.125.92.107/32
|
||||
161.77.143.49/32
|
||||
45.155.203.59/32
|
||||
63.125.90.226/32
|
||||
208.214.161.115/32
|
||||
209.46.2.40/32
|
||||
45.155.202.30/32
|
||||
209.46.2.120/32
|
||||
63.125.90.226/32
|
||||
158.46.163.29/32
|
||||
176.106.63.59/32
|
||||
83.168.86.1/32
|
||||
176.103.230.40/32
|
||||
85.255.180.245/32
|
||||
63.125.93.220/32
|
||||
161.77.71.191/32
|
||||
161.77.70.73/32
|
||||
208.214.160.169/32
|
||||
208.214.165.71/32
|
||||
45.155.203.59/32
|
||||
212.68.177.120/32
|
||||
178.171.120.242/32
|
||||
131.222.219.99/32
|
||||
|
|
@ -659,3 +659,4 @@
|
|||
185.195.19.0/24
|
||||
85.203.15.0/24
|
||||
85.203.36.0/24
|
||||
185.194.178.0/24
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
38.165.232.58/32
|
||||
57.128.102.31/32
|
||||
57.129.2.17/32
|
||||
38.165.232.16/32
|
||||
162.19.19.237/32
|
||||
57.129.25.46/32
|
||||
185.191.204.202/32
|
||||
95.110.229.220/32
|
||||
38.165.228.24/32
|
||||
51.158.206.8/32
|
||||
110.172.145.222/32
|
||||
198.244.176.53/32
|
||||
216.106.183.243/32
|
||||
15.204.97.212/32
|
||||
190.103.179.106/32
|
||||
51.158.204.28/32
|
||||
185.227.138.194/32
|
||||
198.244.231.55/32
|
||||
167.17.69.162/32
|
||||
15.204.97.215/32
|
||||
|
|
|
|||
|
|
@ -39,7 +39,6 @@
|
|||
140.174.187.15/32
|
||||
108.181.68.87/32
|
||||
140.174.187.17/32
|
||||
178.172.217.13/32
|
||||
173.209.63.146/32
|
||||
67.43.236.226/32
|
||||
148.113.221.150/32
|
||||
|
|
@ -161,7 +160,6 @@
|
|||
180.149.230.169/32
|
||||
103.108.230.31/32
|
||||
103.108.230.51/32
|
||||
5.180.44.194/32
|
||||
180.149.230.241/32
|
||||
102.68.86.97/32
|
||||
91.213.233.111/32
|
||||
|
|
@ -184,7 +182,6 @@
|
|||
109.248.149.173/32
|
||||
154.70.207.190/32
|
||||
154.70.207.146/32
|
||||
43.231.113.84/32
|
||||
43.231.114.178/32
|
||||
171.22.254.19/32
|
||||
171.22.254.144/32
|
||||
|
|
@ -209,7 +206,6 @@
|
|||
190.97.163.17/32
|
||||
190.120.229.196/32
|
||||
138.186.143.50/32
|
||||
103.103.0.21/32
|
||||
112.199.95.186/32
|
||||
103.152.255.235/32
|
||||
57.128.235.172/32
|
||||
|
|
@ -240,7 +236,6 @@
|
|||
165.231.211.122/32
|
||||
193.37.255.2/32
|
||||
185.245.85.126/32
|
||||
119.59.98.133/32
|
||||
119.59.98.74/32
|
||||
188.213.34.178/32
|
||||
185.169.64.46/32
|
||||
|
|
|
|||
223
ranges/vpnly.txt
223
ranges/vpnly.txt
|
|
@ -1,217 +1,38 @@
|
|||
46.229.253.218/32
|
||||
195.74.93.252/32
|
||||
78.31.67.36/32
|
||||
89.163.144.107/32
|
||||
46.229.253.197/32
|
||||
89.163.144.21/32
|
||||
89.163.144.37/32
|
||||
46.229.253.212/32
|
||||
5.199.143.248/32
|
||||
195.74.93.75/32
|
||||
213.202.219.29/32
|
||||
195.74.93.73/32
|
||||
5.104.111.246/32
|
||||
195.74.93.74/32
|
||||
46.229.253.195/32
|
||||
78.31.67.70/32
|
||||
208.87.241.235/32
|
||||
108.181.0.171/32
|
||||
208.87.240.255/32
|
||||
108.181.4.69/32
|
||||
208.87.241.221/32
|
||||
208.87.242.23/32
|
||||
108.181.0.21/32
|
||||
108.181.1.241/32
|
||||
208.87.240.19/32
|
||||
208.87.242.199/32
|
||||
108.181.3.149/32
|
||||
208.87.242.111/32
|
||||
108.181.3.145/32
|
||||
108.181.0.109/32
|
||||
45.12.133.12/32
|
||||
104.168.10.152/32
|
||||
64.31.63.35/32
|
||||
104.168.10.60/32
|
||||
45.12.133.22/32
|
||||
94.232.247.205/32
|
||||
104.168.10.210/32
|
||||
94.232.247.207/32
|
||||
64.31.63.233/32
|
||||
64.31.63.245/32
|
||||
104.168.10.80/32
|
||||
64.31.63.133/32
|
||||
104.168.10.61/32
|
||||
45.12.133.18/32
|
||||
104.168.10.168/32
|
||||
46.229.243.197/32
|
||||
46.229.243.200/32
|
||||
108.181.59.34/32
|
||||
46.229.243.199/32
|
||||
46.229.243.201/32
|
||||
108.181.58.33/32
|
||||
46.229.243.203/32
|
||||
46.229.243.178/32
|
||||
80.92.204.6/32
|
||||
80.92.204.74/32
|
||||
45.12.138.188/32
|
||||
80.71.157.209/32
|
||||
80.92.204.84/32
|
||||
45.83.129.161/32
|
||||
80.92.204.82/32
|
||||
45.14.247.113/32
|
||||
80.92.204.73/32
|
||||
80.92.204.75/32
|
||||
80.71.157.218/32
|
||||
80.71.157.239/32
|
||||
80.92.204.85/32
|
||||
80.92.204.57/32
|
||||
80.92.204.42/32
|
||||
80.92.204.62/32
|
||||
208.87.240.255/32
|
||||
108.181.0.171/32
|
||||
108.181.0.21/32
|
||||
108.181.3.145/32
|
||||
208.87.241.235/32
|
||||
208.87.241.221/32
|
||||
108.181.1.241/32
|
||||
208.87.242.111/32
|
||||
208.87.242.23/32
|
||||
108.181.4.69/32
|
||||
208.87.240.19/32
|
||||
108.181.3.149/32
|
||||
108.181.0.109/32
|
||||
208.87.242.199/32
|
||||
45.12.133.18/32
|
||||
45.12.133.12/32
|
||||
94.232.247.207/32
|
||||
94.232.247.205/32
|
||||
45.12.133.22/32
|
||||
46.229.243.200/32
|
||||
46.229.243.201/32
|
||||
46.229.243.203/32
|
||||
46.229.243.197/32
|
||||
46.229.243.199/32
|
||||
46.229.243.178/32
|
||||
45.83.129.161/32
|
||||
45.14.247.113/32
|
||||
80.92.204.62/32
|
||||
80.92.204.74/32
|
||||
80.92.204.57/32
|
||||
80.71.157.239/32
|
||||
80.92.204.73/32
|
||||
80.92.204.82/32
|
||||
80.92.204.42/32
|
||||
80.92.204.75/32
|
||||
45.12.138.188/32
|
||||
80.92.204.84/32
|
||||
80.92.204.6/32
|
||||
80.71.157.218/32
|
||||
80.92.204.62/32
|
||||
80.92.204.85/32
|
||||
80.71.157.209/32
|
||||
208.87.241.235/32
|
||||
108.181.0.21/32
|
||||
208.87.242.23/32
|
||||
108.181.3.145/32
|
||||
108.181.1.241/32
|
||||
108.181.0.109/32
|
||||
208.87.242.111/32
|
||||
208.87.240.19/32
|
||||
208.87.240.255/32
|
||||
108.181.0.171/32
|
||||
208.87.242.199/32
|
||||
108.181.3.149/32
|
||||
208.87.241.221/32
|
||||
108.181.4.69/32
|
||||
94.232.247.205/32
|
||||
45.12.133.22/32
|
||||
45.12.133.12/32
|
||||
94.232.247.207/32
|
||||
45.12.133.18/32
|
||||
46.229.243.199/32
|
||||
46.229.243.197/32
|
||||
46.229.243.203/32
|
||||
46.229.243.201/32
|
||||
46.229.243.200/32
|
||||
46.229.243.178/32
|
||||
80.92.204.74/32
|
||||
80.71.157.218/32
|
||||
80.92.204.75/32
|
||||
80.92.204.82/32
|
||||
80.92.204.62/32
|
||||
80.71.157.239/32
|
||||
80.92.204.57/32
|
||||
45.12.138.188/32
|
||||
80.92.204.6/32
|
||||
45.83.129.161/32
|
||||
80.71.157.209/32
|
||||
80.92.204.85/32
|
||||
80.92.204.42/32
|
||||
80.92.204.73/32
|
||||
80.92.204.84/32
|
||||
45.14.247.113/32
|
||||
208.87.242.23/32
|
||||
108.181.0.21/32
|
||||
208.87.240.255/32
|
||||
208.87.241.221/32
|
||||
208.87.242.199/32
|
||||
108.181.0.171/32
|
||||
108.181.4.69/32
|
||||
108.181.0.109/32
|
||||
208.87.242.111/32
|
||||
108.181.1.241/32
|
||||
108.181.3.149/32
|
||||
108.181.3.145/32
|
||||
208.87.241.235/32
|
||||
208.87.240.19/32
|
||||
45.12.133.12/32
|
||||
45.12.133.22/32
|
||||
45.12.133.18/32
|
||||
94.232.247.207/32
|
||||
94.232.247.205/32
|
||||
46.229.243.203/32
|
||||
46.229.243.197/32
|
||||
46.229.243.201/32
|
||||
46.229.243.178/32
|
||||
46.229.243.199/32
|
||||
46.229.243.200/32
|
||||
80.92.204.6/32
|
||||
80.92.204.82/32
|
||||
80.92.204.62/32
|
||||
80.92.204.84/32
|
||||
80.92.204.85/32
|
||||
80.92.204.73/32
|
||||
80.92.204.42/32
|
||||
80.92.204.74/32
|
||||
45.83.129.161/32
|
||||
80.92.204.75/32
|
||||
80.71.157.218/32
|
||||
45.14.247.113/32
|
||||
80.71.157.239/32
|
||||
80.71.157.209/32
|
||||
45.12.138.188/32
|
||||
80.92.204.57/32
|
||||
108.181.3.149/32
|
||||
208.87.240.19/32
|
||||
108.181.0.109/32
|
||||
208.87.242.111/32
|
||||
108.181.0.21/32
|
||||
208.87.242.23/32
|
||||
108.181.1.241/32
|
||||
108.181.0.171/32
|
||||
208.87.242.199/32
|
||||
208.87.241.235/32
|
||||
208.87.241.221/32
|
||||
108.181.4.69/32
|
||||
108.181.3.145/32
|
||||
45.83.129.161/32
|
||||
80.92.204.73/32
|
||||
80.92.204.82/32
|
||||
80.92.204.75/32
|
||||
208.87.240.255/32
|
||||
108.181.3.145/32
|
||||
208.87.241.235/32
|
||||
108.181.3.149/32
|
||||
208.87.241.221/32
|
||||
108.181.0.109/32
|
||||
108.181.1.241/32
|
||||
208.87.242.199/32
|
||||
208.87.242.111/32
|
||||
108.181.4.69/32
|
||||
108.181.0.171/32
|
||||
208.87.240.19/32
|
||||
208.87.242.23/32
|
||||
94.232.247.205/32
|
||||
45.12.133.12/32
|
||||
94.232.247.207/32
|
||||
45.12.133.22/32
|
||||
94.232.247.207/32
|
||||
45.12.133.18/32
|
||||
46.229.243.200/32
|
||||
46.229.243.197/32
|
||||
45.12.133.12/32
|
||||
46.229.243.201/32
|
||||
46.229.243.178/32
|
||||
46.229.243.200/32
|
||||
46.229.243.203/32
|
||||
46.229.243.199/32
|
||||
46.229.243.197/32
|
||||
46.229.243.178/32
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
const oneClickVpn = await (await fetch('https://1clickvpn.net/api/v1/servers/')).json()
|
||||
Bun.write('ranges/1clickvpn.txt', oneClickVpn.flatMap(v => v.nodes.map(n => n.ip + '/32')).join('\n'))
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
const freeLocationsText = await(await fetch('https://raw.githubusercontent.com/1vpn/browser_extension/b58394076cc61beda2a6cc2292915180db58bc17/src/utils/freeLocations.js')).text()
|
||||
const freeLocations = JSON.parse(freeLocationsText.replace('const freeLocations = ', '').replace('export default freeLocations', '').trim())
|
||||
|
||||
for (const l of Object.entries(freeLocations).filter(h => h[1].isPremium != true)) {
|
||||
for (const h of l[1].hosts) {
|
||||
const v4 = await (await fetch('https://ipinfo.io', {
|
||||
proxy: `https://a2epfq5ugq0u:ptkx3fqg6v7n@${h.hostname}:${h.port}`,
|
||||
headers: {
|
||||
'user-agent': 'curl/8.4.0'
|
||||
}
|
||||
})).json()
|
||||
console.log(v4.ip + '/32')
|
||||
}
|
||||
}
|
||||
|
|
@ -1,79 +0,0 @@
|
|||
import { sleep } from "bun";
|
||||
|
||||
const file = Bun.file('ranges/expressvpn.txt');
|
||||
const writer = file.writer();
|
||||
|
||||
function inetnumToCIDR(value: string): string[] {
|
||||
let [start, end] = value.split(' - ').map(ip =>
|
||||
ip.split('.').reduce((acc, octet) => (acc << 8) | parseInt(octet), 0) >>> 0
|
||||
)
|
||||
|
||||
const cidrs: string[] = []
|
||||
|
||||
while (start <= end) {
|
||||
const maxBits = Math.floor(Math.log2(start & -start)) // largest block start allows
|
||||
const fitBits = Math.floor(Math.log2(end - start + 1)) // largest block size fits
|
||||
const size = Math.min(maxBits, fitBits)
|
||||
const ip = [(start >>> 24) & 255, (start >>> 16) & 255, (start >>> 8) & 255, start & 255].join('.')
|
||||
cidrs.push(`${ip}/${32 - size}`)
|
||||
start += 2 ** size
|
||||
}
|
||||
|
||||
return cidrs
|
||||
}
|
||||
|
||||
const orgs = await (await fetch('https://apps.db.ripe.net/db-web-ui/api/whois/search?abuse-contact=true&ignore404=true&managed-attributes=true&resource-holder=true&type-filter=ORGANISATION&flags=r&offset=0&limit=200&query-string=VPN%20Consumer')).json()
|
||||
const orgId = orgs.objects.object.map(o => o.attributes.attribute.find(a => a.name === 'organisation').value)
|
||||
|
||||
for (const o of [...orgId, 'ORG-ETL39-RIPE']) {
|
||||
const orgReq = await fetch(`https://apps.db.ripe.net/db-web-ui/api/whois/search?abuse-contact=true&ignore404=true&managed-attributes=true&resource-holder=true&type-filter=INETNUM,INET6NUM&inverse-attribute=ORG&flags=r&offset=0&limit=200&query-string=${o}`)
|
||||
if (orgReq.status == 404) {
|
||||
console.log(`no inetnum/inet6num for ${o}`)
|
||||
continue
|
||||
}
|
||||
const org = await orgReq.json()
|
||||
const primaries = org.objects.object.map(o => o['primary-key'].attribute[0])
|
||||
for (const p of primaries) {
|
||||
if (p.name == 'inetnum') {
|
||||
console.log(inetnumToCIDR(p.value).join('\n'))
|
||||
writer.write(inetnumToCIDR(p.value).join('\n') + '\n')
|
||||
} else if (p.name == 'inet6num') {
|
||||
console.log(p.value)
|
||||
writer.write(p.value + '\n')
|
||||
}
|
||||
}
|
||||
|
||||
writer.flush()
|
||||
|
||||
if (orgReq.headers.get('X-Rate-Limit-Remaining') == '1') {
|
||||
console.log('sleeping 5s')
|
||||
await sleep(5000)
|
||||
}
|
||||
}
|
||||
|
||||
let start = 0
|
||||
|
||||
while (true) {
|
||||
const ips = await (await fetch(`https://apps.db.ripe.net/db-web-ui/api/rest/fulltextsearch/select?facet=true&format=xml&hl=true&q=(netname:(%22VPN%5C-Consumer%5C-Network%22))%20AND%20(object-type:inetnum)&start=${start}&wt=json`, {
|
||||
headers: {
|
||||
'Accept': 'application/json, text/plain, */*'
|
||||
}
|
||||
})).json()
|
||||
start = start + 10
|
||||
|
||||
if (!ips.result.docs) break;
|
||||
|
||||
const primaries = ips.result.docs.map(o => o.doc.strs.find(s => s.str.name.startsWith('inetnum')).str)
|
||||
for (const p of primaries) {
|
||||
if (p.name == 'inetnum') {
|
||||
console.log(inetnumToCIDR(p.value).join('\n'))
|
||||
writer.write(inetnumToCIDR(p.value).join('\n') + '\n')
|
||||
} else if (p.name == 'inet6num') {
|
||||
console.log(p.value)
|
||||
writer.write(p.value + '\n')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
writer.flush()
|
||||
writer.end();
|
||||
3312
scripts/planetvpn.ts
3312
scripts/planetvpn.ts
File diff suppressed because it is too large
Load Diff
|
|
@ -1,15 +0,0 @@
|
|||
const json = await (await fetch('https://turbovpn.com/api/mms/serverlist/v1/webext/servers_list', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'X-App-Type': '302',
|
||||
'X-App-Ver-Code': '202507111355',
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
"country": "NL",
|
||||
"user_ip": "1.1.1.1",
|
||||
"os_lang": "en-us",
|
||||
"login_id": "0"
|
||||
})
|
||||
})).json()
|
||||
Bun.write('ranges/turbovpn.txt', json.servers.map(s => s.host_ip + '/32').join('\n'))
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
const post = await (await fetch('https://api-pro.falais.com/rest/v1/security/tokens/accs', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'authorization': 'Bearer FihZXBoQi83OomWPQgj9VqEFPzRsLz6p',
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
"type": "accs",
|
||||
"clientApp": {
|
||||
"name": "URBAN_VPN_BROWSER_EXTENSION"
|
||||
}
|
||||
})
|
||||
})).json()
|
||||
|
||||
const servers = await (await fetch('https://stats.falais.com/api/rest/v2/entrypoints/countries', {
|
||||
headers: {
|
||||
'authorization': `Bearer ${post.value}`,
|
||||
'x-client-app': 'URBAN_VPN_BROWSER_EXTENSION'
|
||||
}
|
||||
})).json()
|
||||
|
||||
Bun.write('ranges/urbanvpn.txt', servers.countries.elements.flatMap(c => c.servers.elements.map(s => s.address.primary.ip + '/32')).join('\n'))
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
import * as fs from 'node:fs'
|
||||
|
||||
async function getARecords(hostname: string): Promise<string[]> {
|
||||
const url = `https://cloudflare-dns.com/dns-query?name=${encodeURIComponent(
|
||||
hostname
|
||||
)}&type=A`;
|
||||
|
||||
const res = await fetch(url, {
|
||||
headers: { Accept: "application/dns-json" },
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`dns query failed: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
|
||||
const body = (await res.json()) as {
|
||||
Status: number;
|
||||
Answer?: Array<{ name: string; type: number; data: string }>;
|
||||
};
|
||||
|
||||
if (body.Status !== 0 || !body.Answer) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return body.Answer.filter((a) => a.type === 1).map((a) => a.data);
|
||||
}
|
||||
|
||||
const hostnames = ['de-hub.freeruproxy.ink', 'us-hub.freeruproxy.ink', 'fr-hub.freeruproxy.ink', 'nl-hub.freeruproxy.ink']
|
||||
for (const h of hostnames) {
|
||||
const records = await getARecords(h)
|
||||
fs.appendFileSync('ranges/vpnly.txt', records.map(r => r + '/32').join('\n') + '\n')
|
||||
}
|
||||
|
|
@ -11,21 +11,40 @@ import { error } from '@/utils/html'
|
|||
import redis from '@/utils/redis';
|
||||
import { parseSlop } from '@/utils/slop';
|
||||
import { checkIpRanges } from '@/utils/ranges';
|
||||
import { getRateLimitSubjects } from '@/utils/rate-limit';
|
||||
import { getRateLimitState, getRateLimitSubjects } from '@/utils/rate-limit';
|
||||
|
||||
const app = new Elysia()
|
||||
const videoIds: Record<string, string> = {}
|
||||
|
||||
const MB_LIMIT = 250
|
||||
const saveKey = (videoId: string) => `save:${videoId}`
|
||||
|
||||
const checkMbLimit = async (subjects: string[], mb?: number): Promise<boolean> => {
|
||||
const keys = subjects.map(subject => `save-mb:${Bun.hash(subject).toString()}`)
|
||||
const currentValues = await Promise.all(keys.map(key => redis.get(key)))
|
||||
const currents = currentValues.map(value => parseInt(value || '0'))
|
||||
const DEFAULT_STORAGE_LIMIT_MESSAGE = 'Daily storage limit reached. Is this an urgent archive? Please email me: admin@preservetube.com'
|
||||
const NEW_VISITOR_STORAGE_LIMIT_MESSAGE = 'You are a new visitor, so your storage limit is lower for the first few hours. Please come back later. </br>Is this an urgent archive? Please email me: admin@preservetube.com'
|
||||
|
||||
if (!mb) return currents.some(current => current >= MB_LIMIT)
|
||||
if (currents.some(current => current + mb > MB_LIMIT)) return true
|
||||
const checkMbLimit = async (subjects: string[], mb?: number): Promise<{ isLimited: boolean, isNewVisitorLimited: boolean }> => {
|
||||
const keys = subjects.map(subject => `save-mb:${Bun.hash(subject).toString()}`)
|
||||
const [currentValues, states] = await Promise.all([
|
||||
Promise.all(keys.map(key => redis.get(key))),
|
||||
Promise.all(subjects.map(subject => getRateLimitState(subject)))
|
||||
])
|
||||
const currents = currentValues.map(value => parseInt(value || '0'))
|
||||
const limitedIndexes = currents
|
||||
.map((current, index) => {
|
||||
const limit = states[index]!.limit
|
||||
if (mb === undefined) return current >= limit ? index : -1
|
||||
return current + mb > limit ? index : -1
|
||||
})
|
||||
.filter(index => index !== -1)
|
||||
|
||||
if (limitedIndexes.length > 0) {
|
||||
return {
|
||||
isLimited: true,
|
||||
isNewVisitorLimited: limitedIndexes.some(index => states[index]!.isNewVisitor)
|
||||
}
|
||||
}
|
||||
if (mb === undefined) {
|
||||
return { isLimited: false, isNewVisitorLimited: false }
|
||||
}
|
||||
|
||||
const pipeline = redis.pipeline()
|
||||
for (const key of keys) {
|
||||
|
|
@ -33,7 +52,7 @@ const checkMbLimit = async (subjects: string[], mb?: number): Promise<boolean> =
|
|||
pipeline.expire(key, 24 * 60 * 60)
|
||||
}
|
||||
await pipeline.exec()
|
||||
return false
|
||||
return { isLimited: false, isNewVisitorLimited: false }
|
||||
}
|
||||
|
||||
const sendError = (ws: any, message: string, close: boolean = true) => {
|
||||
|
|
@ -125,9 +144,9 @@ app.ws('/save', {
|
|||
getRateLimitKey(ws.data.headers['cf-connecting-ip'] || '0.0.0.0'),
|
||||
ws.data.query.rlid
|
||||
)
|
||||
const isLimited = await checkMbLimit(subjects)
|
||||
if (isLimited) {
|
||||
return sendError(ws, 'You have been ratelimited. </br>Is this an urgent archive? Please email me: admin@preservetube.com');
|
||||
const limitStatus = await checkMbLimit(subjects)
|
||||
if (limitStatus.isLimited) {
|
||||
return sendError(ws, limitStatus.isNewVisitorLimited ? NEW_VISITOR_STORAGE_LIMIT_MESSAGE : DEFAULT_STORAGE_LIMIT_MESSAGE);
|
||||
}
|
||||
|
||||
console.log(`saving (${subjects.map(subject => Bun.hash(subject).toString()).join(',')}) - ${ws.data.path} - ${JSON.stringify(ws.data.query)}`)
|
||||
|
|
@ -180,12 +199,12 @@ app.ws('/save', {
|
|||
getRateLimitKey(ws.data.headers['cf-connecting-ip'] || '0.0.0.0'),
|
||||
ws.data.query.rlid
|
||||
)
|
||||
const isMbLimited = await checkMbLimit(subjects, mbsUsed)
|
||||
if (isMbLimited) {
|
||||
const limitStatus = await checkMbLimit(subjects, mbsUsed)
|
||||
if (limitStatus.isLimited) {
|
||||
const file = fs.readdirSync('./videos/').find(f => f.includes(`${videoId}.`))
|
||||
if (file) fs.unlinkSync('./videos/' + file)
|
||||
await cleanup(ws, videoId);
|
||||
return sendError(ws, 'Daily storage limit reached. Is this an urgent archive? Please email me: admin@preservetube.com');
|
||||
return sendError(ws, limitStatus.isNewVisitorLimited ? NEW_VISITOR_STORAGE_LIMIT_MESSAGE : DEFAULT_STORAGE_LIMIT_MESSAGE);
|
||||
}
|
||||
|
||||
const uploadSuccess = await handleUpload(ws, videoId);
|
||||
|
|
@ -263,9 +282,9 @@ app.ws('/savechannel', {
|
|||
getRateLimitKey(ws.data.headers['cf-connecting-ip'] || '0.0.0.0'),
|
||||
ws.data.query.rlid
|
||||
)
|
||||
const isLimited = await checkMbLimit(subjects)
|
||||
if (isLimited) {
|
||||
sendError(ws, 'You have been ratelimited. </br>Is this an urgent archive? Please email me: admin@preservetube.com', false);
|
||||
const limitStatus = await checkMbLimit(subjects)
|
||||
if (limitStatus.isLimited) {
|
||||
sendError(ws, limitStatus.isNewVisitorLimited ? NEW_VISITOR_STORAGE_LIMIT_MESSAGE : DEFAULT_STORAGE_LIMIT_MESSAGE, false);
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -287,11 +306,11 @@ app.ws('/savechannel', {
|
|||
const downloadResult = await downloadVideo(ws, video.video_id);
|
||||
if (!downloadResult.fail) {
|
||||
const mbsUsed = Math.ceil(downloadResult.size / (1024 * 1024))
|
||||
const isMbLimited = await checkMbLimit(subjects, mbsUsed)
|
||||
if (isMbLimited) {
|
||||
const limitStatus = await checkMbLimit(subjects, mbsUsed)
|
||||
if (limitStatus.isLimited) {
|
||||
const file = fs.readdirSync('./videos/').find(f => f.includes(`${video.video_id}.`))
|
||||
if (file) fs.unlinkSync('./videos/' + file)
|
||||
sendError(ws, 'Daily storage limit reached. Is this an urgent archive? Please email me: admin@preservetube.com', false);
|
||||
sendError(ws, limitStatus.isNewVisitorLimited ? NEW_VISITOR_STORAGE_LIMIT_MESSAGE : DEFAULT_STORAGE_LIMIT_MESSAGE, false);
|
||||
break;
|
||||
}
|
||||
await handleUpload(ws, video.video_id, true);
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ const asnBanList: number[] = [
|
|||
206092,
|
||||
137409,
|
||||
262287,
|
||||
29066,
|
||||
|
||||
// tor exits
|
||||
60729,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,10 @@
|
|||
import redis from '@/utils/redis'
|
||||
|
||||
const RATE_LIMIT_COOKIE = 'pt_rlid'
|
||||
const RATE_LIMIT_COOKIE_MAX_AGE = 60 * 60 * 24 * 365
|
||||
const DEFAULT_MB_LIMIT = 250
|
||||
const NEW_IP_MB_LIMIT = 150
|
||||
const NEW_IP_TRUST_WINDOW_MS = 6 * 60 * 60 * 1000 // 6h
|
||||
|
||||
const parseCookieHeader = (cookieHeader?: string): Record<string, string> => {
|
||||
if (!cookieHeader) return {}
|
||||
|
|
@ -25,6 +30,31 @@ export const buildRateLimitCookie = (value: string): string => {
|
|||
return `${RATE_LIMIT_COOKIE}=${encodeURIComponent(value)}; Max-Age=${RATE_LIMIT_COOKIE_MAX_AGE}; Path=/; HttpOnly; SameSite=Lax; Secure`
|
||||
}
|
||||
|
||||
const isIpRateLimitSubject = (subject: string): boolean => subject.startsWith('ip:')
|
||||
|
||||
const getSubjectTrustKey = (subject: string): string => `rate-limit:first-seen:${Bun.hash(subject).toString()}`
|
||||
|
||||
export const getRateLimitState = async (subject: string): Promise<{ limit: number, isNewVisitor: boolean }> => {
|
||||
if (!isIpRateLimitSubject(subject)) {
|
||||
return { limit: DEFAULT_MB_LIMIT, isNewVisitor: false }
|
||||
}
|
||||
|
||||
const now = Date.now()
|
||||
const trustKey = getSubjectTrustKey(subject)
|
||||
await redis.set(trustKey, now.toString(), 'NX')
|
||||
|
||||
const firstSeen = Number(await redis.get(trustKey) || now)
|
||||
if (Number.isNaN(firstSeen)) {
|
||||
return { limit: NEW_IP_MB_LIMIT, isNewVisitor: true }
|
||||
}
|
||||
|
||||
const isNewVisitor = now - firstSeen < NEW_IP_TRUST_WINDOW_MS
|
||||
return {
|
||||
limit: isNewVisitor ? NEW_IP_MB_LIMIT : DEFAULT_MB_LIMIT,
|
||||
isNewVisitor
|
||||
}
|
||||
}
|
||||
|
||||
export const getRateLimitSubjects = (ip: string, visitorId?: string): string[] => {
|
||||
const subjects = new Set<string>()
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue