backend/controller/websocket.js

const fs = require('node:fs')
const crypto = require('node:crypto')
const { RedisRateLimiter } = require('rolling-rate-limiter')
const rtm = require('readable-to-ms')

const upload = require('../utils/upload.js')
const ytdlp = require('../utils/ytdlp.js')
const redis = require('../utils/redis.js')

const validate = require('../utils/validate.js')
const metadata = require('../utils/metadata.js')
const websocket = require('../utils/websocket.js')
const captcha = require("../utils/captcha.js")
const logger = require("../utils/logger.js")

const { PrismaClient } =  require('@prisma/client')
const prisma = new PrismaClient()

const limiter = new RedisRateLimiter({
    client: redis, 
    namespace: 'autodownload:',
    interval: 24 * 60 * 60 * 1000,
    maxInInterval: 5
})

exports.save = async (ws, req) => {
    logger.info({ message: `${req.path} ${JSON.stringify(req.query)}` })

    const id = await validate.validateVideoInput(req.query.url)
    if (id.fail) {
        ws.send(`ERROR - ${id.message}`)
        return ws.close()
    }

    if (await redis.get(id)) {
        ws.send('DATA - Someone is already downloading this video...')
        return ws.close()
    }

    if (await redis.get(`blacklist:${id}`)) {
        ws.send('DATA - You can\'t download that. The video is blacklisted.')
        return ws.close()
    }

    const already = await prisma.videos.findFirst({
        where: {
            id: id
        }
    })

    if (already) return ws.send(`DONE - ${process.env.FRONTEND}/watch?v=${id}`)
    
    ws.send('DATA - This process is automatic. Your video will start archiving shortly.')
    ws.send('CAPTCHA - Solving a cryptographic challenge before downloading.')
    
    ws.on('message', async function(msg) {
        if (msg == 'alive') return 

        if (await redis.get(id) != 'downloading') {
            await redis.set(id, 'downloading', 'EX', 300)
            const confirm = await captcha.checkCaptcha(msg)

            if (confirm) startDownloading()
            else {
                await redis.del(id)
                ws.send('DATA - You little goofy goober tried to mess with the captcha...')
                ws.close()
            }
        } else {
            ws.send('DATA - You already sent captcha reply...')
        }
    })

    async function startDownloading() {
        const download = await ytdlp.downloadVideo(`https://www.youtube.com/watch?v=${id}`, ws, id)
        if (download.fail) {
            await redis.del(id)
            ws.send(`DATA - ${download.message}`)
            ws.close()
        } else {
            const file = fs.readdirSync("videos").find(f => f.includes(id))
            if (file) {    
                ws.send('DATA - Uploading file...')
                const videoUrl = await upload.uploadVideo(`./videos/${id}.mp4`)
                fs.unlinkSync(`./videos/${id}.mp4`)

                const uploaded = await websocket.createDatabaseVideo(id, videoUrl)
                if (uploaded != 'success') {
                    ws.send(`DATA - Error while uploading - ${JSON.stringify(uploaded)}`)
                } else {
                    ws.send(`DONE - ${process.env.FRONTEND}/watch?v=${id}`)
                }
            }
            
            await redis.del(id)
            ws.close();
        }
    }
}

exports.playlist = async (ws, req) => {
    logger.info({ message: `${req.path} ${JSON.stringify(req.query)}` })

    const playlistId = await validate.validatePlaylistInput(req.query.url)
    if (playlistId.fail) {
        ws.send(`ERROR - ${playlistId.message}`)
        return ws.close()
    }

    let status = 'captcha'
    ws.send('DATA - This process is automatic. Your video will start archiving shortly.')
    ws.send('CAPTCHA - Solving a cryptographic challenge before downloading.')
    
    ws.on('message', async function(msg) {
        if (msg == 'alive') return 

        if (status == 'captcha') {
            status = 'downloading'
            const confirm = await captcha.checkCaptcha(msg)

            if (confirm) startDownloading()
            else {
                await redis.del(id)
                ws.send('DATA - You little goofy goober tried to mess with the captcha...')
                ws.close()
            }
        } else {
            ws.send('DATA - You already sent captcha reply...')
        }
    })

    async function startDownloading() {
        const playlist = await metadata.getPlaylistVideos(playlistId)
        for (video of playlist.relatedStreams.slice(0, 5)) {
            if (ws.readyState !== ws.OPEN) {
                return logger.info({ message: `Stopped downloading ${playlistId}, websocket is closed` })
            }

            const id = video.url.match(/[?&]v=([^&]+)/)[1]

            const already = await prisma.videos.findFirst({
                where: {
                    id: id
                }
            })

            if (already) {
                ws.send(`DATA - Already downloaded ${video.title}`)
                await prisma.videos.updateMany({
                    where: {
                        id: id
                    }, 
                    data: {
                        playlist: playlistId
                    }
                })
                continue
            }

            if (await redis.get(id)) {
                ws.send(`DATA - Someone is already downloading ${video.title}, skipping.`)
                continue
            }

            if (await redis.get(`blacklist:${id}`)) {
                ws.send(`DATA - ${video.title} is blacklisted from downloading, skipping`)
                continue
            }

            ws.send(`INFO - Downloading ${video.title}<br><br>`)
            await redis.set(id, 'downloading', 'EX', 300)

            const download = await ytdlp.downloadVideo('https://www.youtube.com' + video.url, ws, id)
            if (download.fail) {
                ws.send(`DATA - ${download.message}`)
                await redis.del(id)
                continue
            } else {
                const file = fs.readdirSync("./videos").find(f => f.includes(id))
                if (file) {
                    try {
                        ws.send(`DATA - Downloaded ${video.title}`)
                        ws.send(`DATA - Uploading ${video.title}`)

                        const videoUrl = await upload.uploadVideo(`./videos/${id}.mp4`)
                        ws.send(`DATA - Uploaded ${video.title}`)
                        fs.unlinkSync(`./videos/${id}.mp4`)

                        await websocket.createDatabaseVideo(id, videoUrl, playlistId)
                        ws.send(`DATA - Created video page for ${video.title}`)
                    } catch (e) {
                        ws.send(`DATA - Failed downloading video ${video.title}. Going to next video`)
                        logger.error(e)
                    }
                } else {
                    ws.send(`DATA - Failed to find file for ${video.title}. Going to next video in the playlist`)
                }

                await redis.del(id)
            }
        }

        ws.send(`DONE - ${process.env.FRONTEND}/playlist?list=${playlistId}`)
    }
}

exports.channel = async (ws, req) => {
    logger.info({ message: `${req.path} ${JSON.stringify(req.query)}` })

    const channelId = await validate.validateChannelInput(req.query.url)
    if (channelId.fail) {
        ws.send(`ERROR - ${channelId.message}`)
        return ws.close()
    }

    let status = 'captcha'
    ws.send('DATA - This process is automatic. Your video will start archiving shortly.')
    ws.send('CAPTCHA - Solving a cryptographic challenge before downloading.')
    
    ws.on('message', async function(msg) {
        if (msg == 'alive') return 
        
        if (status == 'captcha') {
            status = 'downloading'
            const confirm = await captcha.checkCaptcha(msg)

            if (confirm) startDownloading()
            else {
                ws.send('DATA - You little goofy goober tried to mess with the captcha...')
                ws.close()
            }
        } else {
            ws.send('DATA - You already sent captcha reply...')
        }
    })

    async function startDownloading() {
        const videos = await metadata.getChannelVideos(channelId)

        for (const video of videos.slice(0, 5)) {
            if (ws.readyState !== ws.OPEN) {
                return logger.info({ message: `Stopped downloading ${channelId}, websocket is closed` })
            }

            const already = await prisma.videos.findFirst({
                where: {
                    id: video.id
                }
            })

            if (already) {
                ws.send(`DATA - Already downloaded ${video.title.text}`)
                continue
            }

            if (await redis.get(video.id)) {
                ws.send(`DATA - Someone is already downloading ${video.title.text}, skipping.`)
                continue
            }

            if (await redis.get(`blacklist:${video.id}`)) {
                ws.send(`DATA - ${video.title.text} is blacklisted from downloading, skipping`)
                continue
            }
            
            ws.send(`INFO - Downloading ${video.title.text}<br><br>`)
            await redis.set(video.id, 'downloading', 'EX', 300)

            const download = await ytdlp.downloadVideo(`https://www.youtube.com/watch?v=${video.id}`, ws, video.id)
            if (download.fail) {
                ws.send(`DATA - ${download.message}`)
                await redis.del(video.id)
                continue
            } else {
                const file = fs.readdirSync("./videos").find(f => f.includes(video.id))
                if (file) {
                    try {
                        ws.send(`DATA - Downloaded ${video.title.text}`)
                        ws.send(`DATA - Uploading ${video.title.text}`)

                        const videoUrl = await upload.uploadVideo(`./videos/${video.id}.mp4`)
                        ws.send(`DATA - Uploaded ${video.title.text}`)
                        fs.unlinkSync(`./videos/${video.id}.mp4`)

                        await websocket.createDatabaseVideo(video.id, videoUrl)
                        ws.send(`DATA - Created video page for ${video.title.text}`)
                    } catch (e) {
                        ws.send(`DATA - Failed downloading video ${video.title.text}. Going to next video`)
                        logger.error(e)
                    }
                } else {
                    ws.send(`DATA - Failed to find file for ${video.title.text}. Going to next video`)
                }

                await redis.del(video.id)
            }
        }

        ws.send(`DONE - ${process.env.FRONTEND}/channel/${channelId}`)
    }
}

exports.addAutodownload = async (req, res) => {
    const confirm = await captcha.checkCaptcha(req.query.captcha)
    if (!confirm) return res.status(500).send('You little goofy goober tried to mess with the captcha...')

    const channelId = await validate.validateChannelInput(req.query.url)
    if (channelId.fail) {
        return res.status(500).send(channelId.message)
    }

    const already = await prisma.autodownload.findFirst({
        where: {
            channel: channelId
        }
    })

    if (already) {
        res.status(500).send(`This channel is already being automatically downloaded...`)
    } else {
        const ipHash = crypto.createHash('sha256').update(req.headers['x-forwarded-for'] || req.connection.remoteAddress).digest('hex')
        const isLimited = await limiter.limit(ipHash)
    
        if (isLimited) return res.status(420).send(`Hey! You have reached the limit of 5 queued auto-download channels per day. Sadly, hard drives don't grow on trees, so rate limits are necessary. The "Save Channel" feature has no limits, so feel free to use that.<br><br>
        
Are you planning something awesome? Feel free to email me at admin[@]preservetube.com.`)

        await prisma.autodownload.create({
            data: {
                channel: channelId
            }
        })
        res.send('Perfect! Each time this channel uploads their videos will be downloaded')
    }
}