fuck my life. channels migrated to the youtube direct api

controller/video.js

@@ -2,6 +2,7 @@ const { PrismaClient } =  require('@prisma/client')
 const prisma = new PrismaClient()
 
 const DOMPurify = require('isomorphic-dompurify')
+const rtm = require('readable-to-ms')
 const metadata = require('../utils/metadata.js')
 const redis = require('../utils/redis.js')
 
@@ -68,11 +69,15 @@ exports.getChannel = async (req, res) => {
         }
     })
 
-    const processedVideos = videos.map(video => ({
+    const processedVideos = videos.map(video => {
-        id: video.url.replace('/watch?v=', ''),
+        const date = !isNaN(new Date(video.published.text).getTime()) ? new Date(video.published.text) : new Date((new Date()).getTime() - rtm(video.published.text).ms); // life is great.
-        published: new Date(video.uploaded).toISOString().slice(0, 10),
+        return {
-        ...video
+            id: video.id,
-    }));
+            title: video.title.text,
+            thumbnail: video.thumbnails[0].url,
+            published: new Date(date).toISOString().slice(0, 10)
+        }
+    });
 
     archived.forEach(v => {
         const existingVideoIndex = processedVideos.findIndex(video => video.id === v.id);

controller/websocket.js

@@ -1,6 +1,7 @@
 const fs = require('node:fs')
 const crypto = require('node:crypto')
 const { RedisRateLimiter } = require('rolling-rate-limiter')
+const rtm = require('readable-to-ms')
 
 const upload = require('../utils/upload.js')
 const ytdlp = require('../utils/ytdlp.js')
@@ -233,64 +234,62 @@ exports.channel = async (ws, req) => {
     async function startDownloading() {
         const videos = await metadata.getChannelVideos(channelId)
 
-        for (video of videos.slice(0, 5)) {
+        for (const video of videos.slice(0, 5)) {
             if (ws.readyState !== ws.OPEN) {
                 return logger.info({ message: `Stopped downloading ${channelId}, websocket is closed` })
             }
 
-            const id = video.url.match(/[?&]v=([^&]+)/)[1]
-
             const already = await prisma.videos.findFirst({
                 where: {
-                    id: id
+                    id: video.id
                 }
             })
 
             if (already) {
-                ws.send(`DATA - Already downloaded ${video.title}`)
+                ws.send(`DATA - Already downloaded ${video.title.text}`)
                 continue
             }
 
-            if (await redis.get(id)) {
+            if (await redis.get(video.id)) {
-                ws.send(`DATA - Someone is already downloading ${video.title}, skipping.`)
+                ws.send(`DATA - Someone is already downloading ${video.title.text}, skipping.`)
                 continue
             }
 
-            if (await redis.get(`blacklist:${id}`)) {
+            if (await redis.get(`blacklist:${video.id}`)) {
-                ws.send(`DATA - ${video.title} is blacklisted from downloading, skipping`)
+                ws.send(`DATA - ${video.title.text} is blacklisted from downloading, skipping`)
                 continue
             }
             
-            ws.send(`INFO - Downloading ${video.title}<br><br>`)
+            ws.send(`INFO - Downloading ${video.title.text}<br><br>`)
-            await redis.set(id, 'downloading')
+            await redis.set(video.id, 'downloading')
 
-            const download = await ytdlp.downloadVideo('https://www.youtube.com' + video.url, ws, id)
+            const download = await ytdlp.downloadVideo(`https://www.youtube.com/watch?v=${video.id}`, ws, video.id)
             if (download.fail) {
                 ws.send(`DATA - ${download.message}`)
-                await redis.del(id)
+                await redis.del(video.id)
                 continue
             } else {
-                const file = fs.readdirSync("./videos").find(f => f.includes(id))
+                const file = fs.readdirSync("./videos").find(f => f.includes(video.id))
                 if (file) {
                     try {
-                        ws.send(`DATA - Downloaded ${video.title}`)
+                        ws.send(`DATA - Downloaded ${video.title.text}`)
-                        ws.send(`DATA - Uploading ${video.title}`)
+                        ws.send(`DATA - Uploading ${video.title.text}`)
 
-                        const videoUrl = await upload.uploadVideo(`./videos/${id}.mp4`)
+                        const videoUrl = await upload.uploadVideo(`./videos/${video.id}.mp4`)
-                        ws.send(`DATA - Uploaded ${video.title}`)
+                        ws.send(`DATA - Uploaded ${video.title.text}`)
-                        fs.unlinkSync(`./videos/${id}.mp4`)
+                        fs.unlinkSync(`./videos/${video.id}.mp4`)
 
-                        await websocket.createDatabaseVideo(id, videoUrl)
+                        await websocket.createDatabaseVideo(video.id, videoUrl)
-                        ws.send(`DATA - Created video page for ${video.title}`)
+                        ws.send(`DATA - Created video page for ${video.title.text}`)
                     } catch (e) {
-                        ws.send(`DATA - Failed downloading video ${video.title}. Going to next video`)
+                        ws.send(`DATA - Failed downloading video ${video.title.text}. Going to next video`)
                         logger.error(e)
                     }
                 } else {
-                    ws.send(`DATA - Failed to find file for ${video.title}. Going to next video`)
+                    ws.send(`DATA - Failed to find file for ${video.title.text}. Going to next video`)
                 }
 
-                await redis.del(id)
+                await redis.del(video.id)
             }
         }
 

package.json

@@ -18,6 +18,7 @@
     "ioredis": "^5.3.1",
     "isomorphic-dompurify": "^1.0.0",
     "node-fetch": "2",
+    "readable-to-ms": "^1.0.3",
     "rolling-rate-limiter": "^0.4.2",
     "wget-improved": "^3.4.0",
     "winston": "^3.8.2",

utils/metadata.js

@@ -1,31 +1,15 @@
 const { Innertube } = require('youtubei.js');
 const fetch = require('node-fetch')
-const https = require('https')
 
 const maxRetries = 5
 const platforms = ['WEB', 'ANDROID', 'iOS']
 const cobalt = ['http://cobalt-api:9000', 'https://co.wuk.sh', 'http://cobalt-api:9000']
 
-const ignoreSsl = new https.Agent({
-    rejectUnauthorized: false,
-})
-
-async function getInstance() {
-    const instances = await (await fetch('https://api.invidious.io/instances.json?pretty=1', {
-        headers: {
-            'User-Agent': 'Mozilla/5.0 (compatible; PreserveTube/0.0; +https://preservetube.com)'
-        }
-    })).json()
-    const sorted = instances.filter(o => o[1].type == 'https' && o[0] != 'invidious.io.lol' && o[0] != 'invidious.0011.lt')
-    return `https://${sorted[Math.floor(Math.random() * sorted.length)][0]}`
-}
-
 async function getPipedInstance() {
     const instances = await (await fetch('https://piped-instances.kavin.rocks/', {
         headers: {
             'User-Agent': 'Mozilla/5.0 (compatible; PreserveTube/0.0; +https://preservetube.com)'
-        },
+        }
-        agent: ignoreSsl
     })).json()
     return (instances[Math.floor(Math.random() * instances.length)]).api_url
 }
@@ -85,34 +69,29 @@ async function getChannel(id) {
 async function getChannelVideos(id) {
     return new Promise(async (resolve, reject) => {
         try {
-            const videos = []
+            const videos = [];
-            const instance = await getPipedInstance()
+            const yt = await Innertube.create();
-            const json = await (await fetch(`${instance}/channel/${id}`, {
+            const channel = await yt.getChannel(id);
-                headers: {
+            let json = await channel.getVideos();
-                    'User-Agent': 'Mozilla/5.0 (compatible; PreserveTube/0.0; +https://preservetube.com)'
-                }
-            })).json()
-            videos.push(...json.relatedStreams)
-            if (json.nextpage) await getNextPage(json.nextpage)
-            else resolve(videos)
     
-            async function getNextPage(payload) {
+            videos.push(...json.videos);
-                const page = await (await fetch(`${instance}/nextpage/channel/${id}?nextpage=${encodeURIComponent(payload)}`, {
-                    headers: {
-                        'User-Agent': 'Mozilla/5.0 (compatible; PreserveTube/0.0; +https://preservetube.com)'
-                    }
-                })).json()
-                videos.push(...page.relatedStreams)
     
-                if (videos.length >= 60) return resolve(videos)
+            while (json.has_continuation && videos.length < 60) {
-                if (page.nextpage) await getNextPage(page.nextpage)
+                json = await getNextPage(json);
-                else return resolve(videos)
+                videos.push(...json.videos);
             }
     
+            resolve(videos);
+            
         } catch (e) {
-            resolve(false)
+            resolve(false);
+        }
+    });
+    
+    async function getNextPage(json) {
+        const page = await json.getContinuation();
+        return page;
     }
-    })
 }
 
 async function getPlaylistVideos(id) {
@@ -153,4 +132,4 @@ async function getVideoDownload(url, quality) {
     return json
 }
 
-module.exports = { getInstance, getVideoMetadata, getChannel, getChannelVideos, getPlaylistVideos, getVideoDownload }
+module.exports = { getVideoMetadata, getChannel, getChannelVideos, getPlaylistVideos, getVideoDownload }

yarn.lock

@@ -1067,6 +1067,11 @@ readable-stream@^3.4.0, readable-stream@^3.6.0:
     string_decoder "^1.1.1"
     util-deprecate "^1.0.1"
 
+readable-to-ms@^1.0.3:
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/readable-to-ms/-/readable-to-ms-1.0.3.tgz#7a1bc1cdf15a13e0808d488c160ea723fd04e60f"
+  integrity sha512-xxxnoflc1CT6st5azNw5u8m739XkrqrkYgLGTe9noe2OG5ohxz3viFQoOSXN2qdckf79da353uBPHQrsjPuYdQ==
+
 redis-errors@^1.0.0, redis-errors@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad"