adding channel downloading and automatic downloading

controller/websocket.js

@@ -171,4 +171,115 @@ exports.playlist = async (ws, req) => {
 
         ws.send(`DONE - ${process.env.FRONTEND}/playlist?list=${playlistId}`)
     }
+}
+
+exports.channel = async (ws, req) => {
+    logger.info({ message: `${req.path} ${JSON.stringify(req.query)} ${JSON.stringify(req.headers)}` })
+
+    const channelId = await validate.validateChannelInput(req.query.url)
+    if (channelId.fail) {
+        ws.send(`ERROR - ${channelId.message}`)
+        return ws.close()
+    }
+
+    let status = 'captcha'
+    ws.send('CAPTCHA - Please complete the captcha:')
+    
+    ws.on('message', async function(msg) {
+        if (status == 'captcha') {
+            status = 'downloading'
+            const confirm = await captcha.checkCaptcha(msg)
+
+            if (confirm) startDownloading()
+            else {
+                await redis.del(id)
+                ws.send('DATA - You little goofy goober tried to mess with the captcha...')
+                ws.close()
+            }
+        } else {
+            ws.send('DATA - You already sent captcha reply...')
+        }
+    })
+
+    async function startDownloading() {
+        const instance = await metadata.getInstance()
+        const channel = await metadata.getChannelVideos(instance, channelId)
+        for (video of channel.relatedStreams) {
+            const id = video.url.match(/[?&]v=([^&]+)/)[1]
+
+            const already = await prisma.videos.findFirst({
+                where: {
+                    id: id
+                }
+            })
+
+            if (already) {
+                ws.send(`DATA - Already downloaded ${video.title}`)
+                continue
+            }
+
+            if (await redis.get(id)) {
+                ws.send(`DATA - Someone is already downloading ${video.title}, skipping.`)
+                continue
+            }
+
+            ws.send(`INFO - Downloading ${video.title}<br><br>`)
+            await redis.set(id, 'downloading')
+
+            const download = await ytdlp.downloadVideo('https://www.youtube.com' + video.url, ws)
+            if (download.fail) {
+                ws.send(`DATA - ${download.message}`)
+                await redis.del(id)
+                continue
+            } else {
+                await redis.del(id)
+                
+                const file = fs.readdirSync("./videos").find(f => f.includes(id))
+                if (file) {
+                    fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`)
+                    ws.send(`DATA - Downloaded ${video.title}`)
+                    ws.send(`DATA - Uploading ${video.title}`)
+
+                    const videoUrl = await upload.uploadVideo(`./videos/${id}.webm`)
+                    ws.send(`DATA - Uploaded ${video.title}`)
+                    fs.unlinkSync(`./videos/${id}.webm`)
+
+                    await websocket.createDatabaseVideo(id, videoUrl)
+                    ws.send(`DATA - Created video page for ${video.title}`)
+                } else {
+                    ws.send(`DATA - Failed to find file for ${video.title}. Going to next video`)
+                    continue
+                }
+            }
+        }
+
+        ws.send(`DONE - ${process.env.FRONTEND}/channel/${channelId}`)
+    }
+}
+
+exports.addAutodownload = async (req, res) => {
+    const confirm = await captcha.checkCaptcha(req.query.captcha)
+    if (!confirm) res.status(500).send('You little goofy goober tried to mess with the captcha...')
+
+    const channelId = await validate.validateChannelInput(req.query.url)
+    if (channelId.fail) {
+        res.status(500).send(channelId.message)
+    }
+
+    const already = await prisma.autodownload.findFirst({
+        where: {
+            channel: channelId
+        }
+    })
+
+    if (already) {
+        res.status(500).send(`This channel is already being automatically downloaded...`)
+    } else {
+        await prisma.autodownload.create({
+            data: {
+                channel: channelId
+            }
+        })
+        res.send('Perfect! Each time this channel uploads their videos will be downloaded')
+    }
 }

index.js

@@ -4,6 +4,7 @@ const express = require('express')
 const cors = require('cors')
 
 const logger = require('./utils/logger.js')
+const auto = require('./utils/auto.js')
 
 const latestController = require('./controller/latest.js')
 const videoController = require('./controller/video.js')
@@ -30,6 +31,12 @@ app.get('/transparency/:id', transparencyController.getReport)
 
 app.ws('/save', websocketController.save)
 app.ws('/saveplaylist', websocketController.playlist)
+app.ws('/savechannel', websocketController.channel)
+app.get('/autodownload', websocketController.addAutodownload)
+
+setInterval(() => {
+  auto.handleCheck()
+}, 300000)
 
 process.on('uncaughtException', err => {
   logger.info({ message: `Error: ${err.message}` })

prisma/schema.prisma

@@ -34,4 +34,9 @@ model reports {
   title     String
   details   String
   date      DateTime  @default(now())
+}
+
+model autodownload {
+  uuid      String    @id @default(uuid())
+  channel   String
 }

utils/auto.js

@@ -0,0 +1,61 @@
+const logger = require("./logger.js");
+const metadata = require("./metadata.js")
+const ytdlp = require("./ytdlp.js")
+const redis = require("./redis.js")
+const websocket = require("./websocket.js")
+
+const { PrismaClient } =  require('@prisma/client')
+const prisma = new PrismaClient()
+
+async function handleCheck() {
+    const channels = await prisma.autodownload.findMany()
+
+    for (c of channels) {
+        await handleDownload(c.channel)
+    }
+}
+
+async function handleDownload(channelId) {
+    logger.info({ message: `Checking ${channelId} for new videos...` })
+
+    const instance = await metadata.getInstance()
+    const channel = await metadata.getChannelVideos(instance, channelId)
+    for (video of channel.relatedStreams) {
+        const id = video.url.match(/[?&]v=([^&]+)/)[1]
+
+        const already = await prisma.videos.findFirst({
+            where: {
+                id: id
+            }
+        })
+
+        if (already) continue
+        logger.info({ message: `Starting to download ${video.title}, ${id}` })
+
+        const download = await ytdlp.downloadVideo('https://www.youtube.com' + video.url)
+        if (download.fail) {
+            logger.info({ message: `Failed downloading ${video.title}, ${id} -> ${download.message}` })
+            await redis.del(id)
+            continue
+        } else {
+            await redis.del(id)
+
+            const file = fs.readdirSync("./videos").find(f => f.includes(id))
+            if (file) {
+                fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`)
+                logger.info({ message: `Downloaded ${video.title}, ${id}` })
+
+                const videoUrl = await upload.uploadVideo(`./videos/${id}.webm`)
+                logger.info({ message: `Uploaded ${video.title}, ${id}` })
+                fs.unlinkSync(`./videos/${id}.webm`)
+
+                await websocket.createDatabaseVideo(id, videoUrl)
+            } else {
+                logger.info({ message: `Couldn't find file for ${video.title}, ${id}` })
+                continue
+            }
+        }
+    }
+}
+
+module.exports = { handleCheck }

utils/ytdlp.js

@@ -8,7 +8,7 @@ async function downloadVideo(url, ws) {
             const msg = data.toString().trim()
             if (!msg) return 
     
-            ws.send(`DATA - ${msg}`)
+            if (ws) ws.send(`DATA - ${msg}`)
         })
 
         child.on("close", async (code, signal) => {