also add cookie/localStorage based ratelimited to avoid (stupid) free

vpn abusers

src/router/html.ts

@@ -2,6 +2,11 @@ import { Elysia } from 'elysia';
 import { m, eta, error } from '@/utils/html'
 import healthStatus from '@/utils/health';
 import { checkIpRanges } from '@/utils/ranges';
+import {
+  buildRateLimitCookie,
+  createRateLimitCookieValue,
+  getRateLimitCookie,
+} from '@/utils/rate-limit';
 const app = new Elysia()
 
 app.get('/', async ({ set }) => {
@@ -29,15 +34,20 @@ app.get('/save', async ({ query: { url }, set, headers, error }) => {
     websocket = process.env.ALTERNATIVE_WEBSOCKET!
   }
   const ytPatched = (process.env.YT_PATCHED || '').toLowerCase() === 'true' || process.env.YT_PATCHED === '1'
+  const rateLimitId = getRateLimitCookie(headers.cookie) || createRateLimitCookieValue()
+  if (!getRateLimitCookie(headers.cookie)) {
+    set.headers['Set-Cookie'] = buildRateLimitCookie(rateLimitId)
+  }
 
   set.headers['Content-Type'] = 'text/html; charset=utf-8'
-  return error(412, await m(eta.render('./save', {
+  return await m(eta.render('./save', {
     title: 'Save Video | PreserveTube',
     websocket,
     sitekey: process.env.SITEKEY,
     url,
-    ytPatched
+    ytPatched,
-  })))
+    rateLimitId
+  }))
 })
 
 app.get('/savechannel', async ({ query: { url }, set, headers, error }) => {
@@ -46,22 +56,27 @@ app.get('/savechannel', async ({ query: { url }, set, headers, error }) => {
   const ranges = await checkIpRanges(headers['cf-connecting-ip'] || headers['x-forwarded-for'] || '')
   if (ranges.blocked) {
     set.headers['Content-Type'] = 'text/html; charset=utf-8'
-    return await m(eta.render('./blocked', {
+    return error(412, await m(eta.render('./blocked', {
       title: 'Blocked | PreserveTube'
-    }))
+    })))
   }
 
   let websocket = process.env.WEBSOCKET
   if (healthStatus[process.env.METADATA!] != 'healthy') {
     websocket = process.env.ALTERNATIVE_WEBSOCKET!
   }
+  const rateLimitId = getRateLimitCookie(headers.cookie) || createRateLimitCookieValue()
+  if (!getRateLimitCookie(headers.cookie)) {
+    set.headers['Set-Cookie'] = buildRateLimitCookie(rateLimitId)
+  }
 
   set.headers['Content-Type'] = 'text/html; charset=utf-8'
   return await m(eta.render('./savechannel', {
     title: 'Save Channel | PreserveTube',
     websocket,
     sitekey: process.env.SITEKEY,
-    url
+    url,
+    rateLimitId
   }))
 })
 

src/router/websocket.ts

@@ -11,6 +11,7 @@ import { error } from '@/utils/html'
 import redis from '@/utils/redis';
 import { parseSlop } from '@/utils/slop';
 import { checkIpRanges } from '@/utils/ranges';
+import { getRateLimitSubjects } from '@/utils/rate-limit';
 
 const app = new Elysia()
 const videoIds: Record<string, string> = {}
@@ -18,15 +19,19 @@ const videoIds: Record<string, string> = {}
 const MB_LIMIT = 250
 const saveKey = (videoId: string) => `save:${videoId}`
 
-const checkMbLimit = async (hash: string, mb?: number): Promise<boolean> => {
+const checkMbLimit = async (subjects: string[], mb?: number): Promise<boolean> => {
-  const key = `save-mb:${hash}`
+  const keys = subjects.map(subject => `save-mb:${Bun.hash(subject).toString()}`)
-  const current = parseInt(await redis.get(key) || '0')
+  const currentValues = await Promise.all(keys.map(key => redis.get(key)))
-  if (!mb) return current >= MB_LIMIT
+  const currents = currentValues.map(value => parseInt(value || '0'))
-  if (current + mb > MB_LIMIT) return true
+
+  if (!mb) return currents.some(current => current >= MB_LIMIT)
+  if (currents.some(current => current + mb > MB_LIMIT)) return true
 
   const pipeline = redis.pipeline()
+  for (const key of keys) {
     pipeline.incrby(key, mb)
     pipeline.expire(key, 24 * 60 * 60)
+  }
   await pipeline.exec()
   return false
 }
@@ -89,7 +94,8 @@ const getRateLimitKey = (ip: string): string => {
 
 app.ws('/save', {
   query: t.Object({
-    url: t.String()
+    url: t.String(),
+    rlid: t.Optional(t.String())
   }),
   body: t.String(),
   open: async (ws) => {
@@ -115,13 +121,16 @@ app.ws('/save', {
       ws.send(`DONE - ${process.env.FRONTEND}/watch?v=${videoId}`)
       ws.close()
     } else {
-      const hash = Bun.hash(getRateLimitKey(ws.data.headers['cf-connecting-ip'] || '0.0.0.0'))
+      const subjects = getRateLimitSubjects(
-      const isLimited = await checkMbLimit(hash.toString())
+        getRateLimitKey(ws.data.headers['cf-connecting-ip'] || '0.0.0.0'),
+        ws.data.query.rlid
+      )
+      const isLimited = await checkMbLimit(subjects)
       if (isLimited) {
         return sendError(ws, 'You have been ratelimited. </br>Is this an urgent archive? Please email me: admin@preservetube.com');
       }
 
-      console.log(`saving (${hash}) - ${ws.data.path} - ${JSON.stringify(ws.data.query)}`)
+      console.log(`saving (${subjects.map(subject => Bun.hash(subject).toString()).join(',')}) - ${ws.data.path} - ${JSON.stringify(ws.data.query)}`)
       ws.send('DATA - This process is automatic. Your video will start archiving shortly.')
       ws.send('CAPTCHA - Solving a cryptographic challenge before downloading.')
       videoIds[ws.id] = videoId
@@ -167,8 +176,11 @@ app.ws('/save', {
       }
 
       const mbsUsed = Math.ceil(downloadResult.size / (1024 * 1024))
-      const hash = Bun.hash(getRateLimitKey(ws.data.headers['cf-connecting-ip'] || '0.0.0.0'))
+      const subjects = getRateLimitSubjects(
-      const isMbLimited = await checkMbLimit(hash.toString(), mbsUsed)
+        getRateLimitKey(ws.data.headers['cf-connecting-ip'] || '0.0.0.0'),
+        ws.data.query.rlid
+      )
+      const isMbLimited = await checkMbLimit(subjects, mbsUsed)
       if (isMbLimited) {
         const file = fs.readdirSync('./videos/').find(f => f.includes(`${videoId}.`))
         if (file) fs.unlinkSync('./videos/' + file)
@@ -193,7 +205,8 @@ app.ws('/save', {
 
 app.ws('/savechannel', {
   query: t.Object({
-    url: t.String()
+    url: t.String(),
+    rlid: t.Optional(t.String())
   }),
   body: t.String(),
   open: async (ws) => {
@@ -246,14 +259,17 @@ app.ws('/savechannel', {
         .executeTakeFirst()
       if (already) continue
 
-      const hash = Bun.hash(getRateLimitKey(ws.data.headers['cf-connecting-ip'] || '0.0.0.0'))
+      const subjects = getRateLimitSubjects(
-      const isLimited = await checkMbLimit(hash.toString())
+        getRateLimitKey(ws.data.headers['cf-connecting-ip'] || '0.0.0.0'),
+        ws.data.query.rlid
+      )
+      const isLimited = await checkMbLimit(subjects)
       if (isLimited) {
         sendError(ws, 'You have been ratelimited. </br>Is this an urgent archive? Please email me: admin@preservetube.com', false);
         break;
       }
 
-      console.log(`saving (${hash}) - ${ws.data.path} - ${video.video_id}`)
+      console.log(`saving (${subjects.map(subject => Bun.hash(subject).toString()).join(',')}) - ${ws.data.path} - ${video.video_id}`)
 
       const isSlop = await parseSlop(video.video_id, video.title.text, 
         video.description_snippet?.text || '', channelId)
@@ -271,7 +287,7 @@ app.ws('/savechannel', {
       const downloadResult = await downloadVideo(ws, video.video_id);
       if (!downloadResult.fail) {
         const mbsUsed = Math.ceil(downloadResult.size / (1024 * 1024))
-        const isMbLimited = await checkMbLimit(hash.toString(), mbsUsed)
+        const isMbLimited = await checkMbLimit(subjects, mbsUsed)
         if (isMbLimited) {
           const file = fs.readdirSync('./videos/').find(f => f.includes(`${video.video_id}.`))
           if (file) fs.unlinkSync('./videos/' + file)

src/templates/save.eta

@@ -45,6 +45,7 @@
   const url = "<%= it.url %>"
   const websocket = "<%= it.websocket %>"
   const sitekey = "<%= it.sitekey %>"
+  const initialRateLimitId = "<%= it.rateLimitId %>"
   let ws = null
   const h1 = document.getElementById("title")
   const h3 = document.getElementById("bottom")
@@ -53,8 +54,33 @@
   const acceptInstructions = document.getElementById("accept-instructions")
   const declineInstructions = document.getElementById("decline-instructions")
 
+  function getRateLimitId() {
+    if (initialRateLimitId) {
+      return initialRateLimitId
+    }
+
+    try {
+      const stored = localStorage.getItem("pt_rlid")
+      if (stored) {
+        return stored
+      }
+    } catch {
+    }
+
+    return initialRateLimitId
+  }
+
+  function persistRateLimitId(rateLimitId) {
+    try {
+      localStorage.setItem("pt_rlid", rateLimitId)
+    } catch {
+    }
+  }
+
   function startSaveFlow() {
-    ws = new WebSocket(`${websocket}/save?url=${encodeURIComponent(url)}`)
+    const rateLimitId = getRateLimitId()
+    persistRateLimitId(rateLimitId)
+    ws = new WebSocket(`${websocket}/save?url=${encodeURIComponent(url)}&rlid=${encodeURIComponent(rateLimitId)}`)
 
     ws.onopen = function () {
       document.querySelector("footer").innerHTML = `page rendered by <code><%= hostname %></code>, connected via <code>${websocket.split('-')[1].split('.')[0]}</code>`
@@ -145,6 +171,7 @@
   }, 2000)
 
   runFirstSaveGate()
+  persistRateLimitId(initialRateLimitId)
 </script>
 <% } %>
 

src/templates/savechannel.eta

@@ -13,7 +13,27 @@
   const url = "<%= it.url %>"
   const websocket = "<%= it.websocket %>"
   const sitekey = "<%= it.sitekey %>"
-  const ws = new WebSocket(`${websocket}/savechannel?url=${encodeURIComponent(url)}`)
+  const initialRateLimitId = "<%= it.rateLimitId %>"
+  function getRateLimitId() {
+    if (initialRateLimitId) {
+      return initialRateLimitId
+    }
+
+    try {
+      const stored = localStorage.getItem("pt_rlid")
+      if (stored) {
+        return stored
+      }
+    } catch {
+    }
+
+    return initialRateLimitId
+  }
+  try {
+    localStorage.setItem("pt_rlid", initialRateLimitId)
+  } catch {
+  }
+  const ws = new WebSocket(`${websocket}/savechannel?url=${encodeURIComponent(url)}&rlid=${encodeURIComponent(getRateLimitId())}`)
   const h1 = document.getElementById("title")
   const h3 = document.getElementById("bottom")
   const captcha = document.getElementById("captcha")