335 lines
12 KiB
JavaScript
335 lines
12 KiB
JavaScript
const fs = require('node:fs')
|
|
const crypto = require('node:crypto')
|
|
const { RedisRateLimiter } = require('rolling-rate-limiter')
|
|
const rtm = require('readable-to-ms')
|
|
|
|
const upload = require('../utils/upload.js')
|
|
const ytdlp = require('../utils/ytdlp.js')
|
|
const redis = require('../utils/redis.js')
|
|
|
|
const validate = require('../utils/validate.js')
|
|
const metadata = require('../utils/metadata.js')
|
|
const websocket = require('../utils/websocket.js')
|
|
const captcha = require("../utils/captcha.js")
|
|
const logger = require("../utils/logger.js")
|
|
|
|
const { PrismaClient } = require('@prisma/client')
|
|
const prisma = new PrismaClient()
|
|
|
|
const limiter = new RedisRateLimiter({
|
|
client: redis,
|
|
namespace: 'autodownload:',
|
|
interval: 24 * 60 * 60 * 1000,
|
|
maxInInterval: 5
|
|
})
|
|
|
|
exports.save = async (ws, req) => {
|
|
logger.info({ message: `${req.path} ${JSON.stringify(req.query)}` })
|
|
|
|
const id = await validate.validateVideoInput(req.query.url)
|
|
if (id.fail) {
|
|
ws.send(`ERROR - ${id.message}`)
|
|
return ws.close()
|
|
}
|
|
|
|
if (await redis.get(id)) {
|
|
ws.send('DATA - Someone is already downloading this video...')
|
|
return ws.close()
|
|
}
|
|
|
|
if (await redis.get(`blacklist:${id}`)) {
|
|
ws.send('DATA - You can\'t download that. The video is blacklisted.')
|
|
return ws.close()
|
|
}
|
|
|
|
const already = await prisma.videos.findFirst({
|
|
where: {
|
|
id: id
|
|
}
|
|
})
|
|
|
|
if (already) return ws.send(`DONE - ${process.env.FRONTEND}/watch?v=${id}`)
|
|
|
|
ws.send('DATA - This process is automatic. Your video will start archiving shortly.')
|
|
ws.send('CAPTCHA - Solving a cryptographic challenge before downloading.')
|
|
|
|
ws.on('message', async function(msg) {
|
|
if (msg == 'alive') return
|
|
|
|
if (await redis.get(id) != 'downloading') {
|
|
await redis.set(id, 'downloading', 'EX', 300)
|
|
const confirm = await captcha.checkCaptcha(msg)
|
|
|
|
if (confirm) startDownloading()
|
|
else {
|
|
await redis.del(id)
|
|
ws.send('DATA - You little goofy goober tried to mess with the captcha...')
|
|
ws.close()
|
|
}
|
|
} else {
|
|
ws.send('DATA - You already sent captcha reply...')
|
|
}
|
|
})
|
|
|
|
async function startDownloading() {
|
|
const download = await ytdlp.downloadVideo(`https://www.youtube.com/watch?v=${id}`, ws, id)
|
|
if (download.fail) {
|
|
await redis.del(id)
|
|
ws.send(`DATA - ${download.message}`)
|
|
ws.close()
|
|
} else {
|
|
const file = fs.readdirSync("videos").find(f => f.includes(id))
|
|
if (file) {
|
|
ws.send('DATA - Uploading file...')
|
|
const videoUrl = await upload.uploadVideo(`./videos/${id}.mp4`)
|
|
fs.unlinkSync(`./videos/${id}.mp4`)
|
|
|
|
const uploaded = await websocket.createDatabaseVideo(id, videoUrl)
|
|
if (uploaded != 'success') {
|
|
ws.send(`DATA - Error while uploading - ${JSON.stringify(uploaded)}`)
|
|
} else {
|
|
ws.send(`DONE - ${process.env.FRONTEND}/watch?v=${id}`)
|
|
}
|
|
}
|
|
|
|
await redis.del(id)
|
|
ws.close();
|
|
}
|
|
}
|
|
}
|
|
|
|
exports.playlist = async (ws, req) => {
|
|
logger.info({ message: `${req.path} ${JSON.stringify(req.query)}` })
|
|
|
|
const playlistId = await validate.validatePlaylistInput(req.query.url)
|
|
if (playlistId.fail) {
|
|
ws.send(`ERROR - ${playlistId.message}`)
|
|
return ws.close()
|
|
}
|
|
|
|
let status = 'captcha'
|
|
ws.send('DATA - This process is automatic. Your video will start archiving shortly.')
|
|
ws.send('CAPTCHA - Solving a cryptographic challenge before downloading.')
|
|
|
|
ws.on('message', async function(msg) {
|
|
if (msg == 'alive') return
|
|
|
|
if (status == 'captcha') {
|
|
status = 'downloading'
|
|
const confirm = await captcha.checkCaptcha(msg)
|
|
|
|
if (confirm) startDownloading()
|
|
else {
|
|
await redis.del(id)
|
|
ws.send('DATA - You little goofy goober tried to mess with the captcha...')
|
|
ws.close()
|
|
}
|
|
} else {
|
|
ws.send('DATA - You already sent captcha reply...')
|
|
}
|
|
})
|
|
|
|
async function startDownloading() {
|
|
const playlist = await metadata.getPlaylistVideos(playlistId)
|
|
for (video of playlist.relatedStreams.slice(0, 5)) {
|
|
if (ws.readyState !== ws.OPEN) {
|
|
return logger.info({ message: `Stopped downloading ${playlistId}, websocket is closed` })
|
|
}
|
|
|
|
const id = video.url.match(/[?&]v=([^&]+)/)[1]
|
|
|
|
const already = await prisma.videos.findFirst({
|
|
where: {
|
|
id: id
|
|
}
|
|
})
|
|
|
|
if (already) {
|
|
ws.send(`DATA - Already downloaded ${video.title}`)
|
|
await prisma.videos.updateMany({
|
|
where: {
|
|
id: id
|
|
},
|
|
data: {
|
|
playlist: playlistId
|
|
}
|
|
})
|
|
continue
|
|
}
|
|
|
|
if (await redis.get(id)) {
|
|
ws.send(`DATA - Someone is already downloading ${video.title}, skipping.`)
|
|
continue
|
|
}
|
|
|
|
if (await redis.get(`blacklist:${id}`)) {
|
|
ws.send(`DATA - ${video.title} is blacklisted from downloading, skipping`)
|
|
continue
|
|
}
|
|
|
|
ws.send(`INFO - Downloading ${video.title}<br><br>`)
|
|
await redis.set(id, 'downloading', 'EX', 300)
|
|
|
|
const download = await ytdlp.downloadVideo('https://www.youtube.com' + video.url, ws, id)
|
|
if (download.fail) {
|
|
ws.send(`DATA - ${download.message}`)
|
|
await redis.del(id)
|
|
continue
|
|
} else {
|
|
const file = fs.readdirSync("./videos").find(f => f.includes(id))
|
|
if (file) {
|
|
try {
|
|
ws.send(`DATA - Downloaded ${video.title}`)
|
|
ws.send(`DATA - Uploading ${video.title}`)
|
|
|
|
const videoUrl = await upload.uploadVideo(`./videos/${id}.mp4`)
|
|
ws.send(`DATA - Uploaded ${video.title}`)
|
|
fs.unlinkSync(`./videos/${id}.mp4`)
|
|
|
|
await websocket.createDatabaseVideo(id, videoUrl, playlistId)
|
|
ws.send(`DATA - Created video page for ${video.title}`)
|
|
} catch (e) {
|
|
ws.send(`DATA - Failed downloading video ${video.title}. Going to next video`)
|
|
logger.error(e)
|
|
}
|
|
} else {
|
|
ws.send(`DATA - Failed to find file for ${video.title}. Going to next video in the playlist`)
|
|
}
|
|
|
|
await redis.del(id)
|
|
}
|
|
}
|
|
|
|
ws.send(`DONE - ${process.env.FRONTEND}/playlist?list=${playlistId}`)
|
|
}
|
|
}
|
|
|
|
exports.channel = async (ws, req) => {
|
|
logger.info({ message: `${req.path} ${JSON.stringify(req.query)}` })
|
|
|
|
const channelId = await validate.validateChannelInput(req.query.url)
|
|
if (channelId.fail) {
|
|
ws.send(`ERROR - ${channelId.message}`)
|
|
return ws.close()
|
|
}
|
|
|
|
let status = 'captcha'
|
|
ws.send('DATA - This process is automatic. Your video will start archiving shortly.')
|
|
ws.send('CAPTCHA - Solving a cryptographic challenge before downloading.')
|
|
|
|
ws.on('message', async function(msg) {
|
|
if (msg == 'alive') return
|
|
|
|
if (status == 'captcha') {
|
|
status = 'downloading'
|
|
const confirm = await captcha.checkCaptcha(msg)
|
|
|
|
if (confirm) startDownloading()
|
|
else {
|
|
ws.send('DATA - You little goofy goober tried to mess with the captcha...')
|
|
ws.close()
|
|
}
|
|
} else {
|
|
ws.send('DATA - You already sent captcha reply...')
|
|
}
|
|
})
|
|
|
|
async function startDownloading() {
|
|
const videos = await metadata.getChannelVideos(channelId)
|
|
|
|
for (const video of videos.slice(0, 5)) {
|
|
if (ws.readyState !== ws.OPEN) {
|
|
return logger.info({ message: `Stopped downloading ${channelId}, websocket is closed` })
|
|
}
|
|
|
|
const already = await prisma.videos.findFirst({
|
|
where: {
|
|
id: video.id
|
|
}
|
|
})
|
|
|
|
if (already) {
|
|
ws.send(`DATA - Already downloaded ${video.title.text}`)
|
|
continue
|
|
}
|
|
|
|
if (await redis.get(video.id)) {
|
|
ws.send(`DATA - Someone is already downloading ${video.title.text}, skipping.`)
|
|
continue
|
|
}
|
|
|
|
if (await redis.get(`blacklist:${video.id}`)) {
|
|
ws.send(`DATA - ${video.title.text} is blacklisted from downloading, skipping`)
|
|
continue
|
|
}
|
|
|
|
ws.send(`INFO - Downloading ${video.title.text}<br><br>`)
|
|
await redis.set(video.id, 'downloading', 'EX', 300)
|
|
|
|
const download = await ytdlp.downloadVideo(`https://www.youtube.com/watch?v=${video.id}`, ws, video.id)
|
|
if (download.fail) {
|
|
ws.send(`DATA - ${download.message}`)
|
|
await redis.del(video.id)
|
|
continue
|
|
} else {
|
|
const file = fs.readdirSync("./videos").find(f => f.includes(video.id))
|
|
if (file) {
|
|
try {
|
|
ws.send(`DATA - Downloaded ${video.title.text}`)
|
|
ws.send(`DATA - Uploading ${video.title.text}`)
|
|
|
|
const videoUrl = await upload.uploadVideo(`./videos/${video.id}.mp4`)
|
|
ws.send(`DATA - Uploaded ${video.title.text}`)
|
|
fs.unlinkSync(`./videos/${video.id}.mp4`)
|
|
|
|
await websocket.createDatabaseVideo(video.id, videoUrl)
|
|
ws.send(`DATA - Created video page for ${video.title.text}`)
|
|
} catch (e) {
|
|
ws.send(`DATA - Failed downloading video ${video.title.text}. Going to next video`)
|
|
logger.error(e)
|
|
}
|
|
} else {
|
|
ws.send(`DATA - Failed to find file for ${video.title.text}. Going to next video`)
|
|
}
|
|
|
|
await redis.del(video.id)
|
|
}
|
|
}
|
|
|
|
ws.send(`DONE - ${process.env.FRONTEND}/channel/${channelId}`)
|
|
}
|
|
}
|
|
|
|
exports.addAutodownload = async (req, res) => {
|
|
const confirm = await captcha.checkCaptcha(req.query.captcha)
|
|
if (!confirm) return res.status(500).send('You little goofy goober tried to mess with the captcha...')
|
|
|
|
const channelId = await validate.validateChannelInput(req.query.url)
|
|
if (channelId.fail) {
|
|
return res.status(500).send(channelId.message)
|
|
}
|
|
|
|
const already = await prisma.autodownload.findFirst({
|
|
where: {
|
|
channel: channelId
|
|
}
|
|
})
|
|
|
|
if (already) {
|
|
res.status(500).send(`This channel is already being automatically downloaded...`)
|
|
} else {
|
|
const ipHash = crypto.createHash('sha256').update(req.headers['x-forwarded-for'] || req.connection.remoteAddress).digest('hex')
|
|
const isLimited = await limiter.limit(ipHash)
|
|
|
|
if (isLimited) return res.status(420).send(`Hey! You have reached the limit of 5 queued auto-download channels per day. Sadly, hard drives don't grow on trees, so rate limits are necessary. The "Save Channel" feature has no limits, so feel free to use that.<br><br>
|
|
|
|
Are you planning something awesome? Feel free to email me at admin[@]preservetube.com.`)
|
|
|
|
await prisma.autodownload.create({
|
|
data: {
|
|
channel: channelId
|
|
}
|
|
})
|
|
res.send('Perfect! Each time this channel uploads their videos will be downloaded')
|
|
}
|
|
} |