preservetube
/
backend
mirror of https://github.com/preservetube/backend.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ratelimits & some more stuff

This commit is contained in:
localhost 2024-02-15 16:13:12 +01:00
parent 8a4602fbfd
commit 2b4e8d25f6
6 changed files with 195 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
controller/latest.js
View File

@ -1,24 +1,35 @@
const { PrismaClient } = require('@prisma/client') const { PrismaClient } = require('@prisma/client')
const redis = require('../utils/redis.js')
const prisma = new PrismaClient() const prisma = new PrismaClient()
exports.getLatest = async (req, res) => { exports.getLatest = async (req, res) => {
res.json(await prisma.videos.findMany({ let json
take: 30, const cached = await redis.get('latest')
orderBy: [
{ if (cached) {
archived: 'desc' json = JSON.parse(cached)
} else {
json = await prisma.videos.findMany({
take: 90,
orderBy: [
{
archived: 'desc'
}
],
select: {
id: true,
title: true,
thumbnail: true,
published: true,
archived: true,
channel: true,
channelId: true,
channelAvatar: true,
channelVerified: true
} }
], })
select: { await redis.set('latest', JSON.stringify(json), 'EX', 3600)
id: true, }
title: true,
thumbnail: true, res.json(json)
published: true,
archived: true,
channel: true,
channelId: true,
channelAvatar: true,
channelVerified: true
}
}))
} }

18
controller/search.js
View File

@ -1,8 +1,22 @@
const crypto = require('node:crypto')
const validate = require('../utils/validate.js') const validate = require('../utils/validate.js')
const redis = require('../utils/redis.js')
const { RedisRateLimiter } = require('rolling-rate-limiter')
const { PrismaClient } = require('@prisma/client') const { PrismaClient } = require('@prisma/client')
const prisma = new PrismaClient() const prisma = new PrismaClient()
const limiter = new RedisRateLimiter({
client: redis,
namespace: 'search:',
interval: 5 * 60 * 1000,
maxInInterval: 5
})
exports.searchVideo = async (req, res) => { exports.searchVideo = async (req, res) => {
const ipHash = crypto.createHash('sha256').update(req.headers['x-userip'] || '0.0.0.0').digest('hex')
const isLimited = await limiter.limit(ipHash)
if (isLimited) return res.status(429).send('error-You have been ratelimited.')
const id = await validate.validateVideoInput(req.query.search) const id = await validate.validateVideoInput(req.query.search)
if (id.fail) { if (id.fail) {
const videos = await prisma.videos.findMany({ const videos = await prisma.videos.findMany({
@ -24,7 +38,7 @@ exports.searchPlaylist = async (req, res) => {
if (id.fail) { if (id.fail) {
res.status(500).send(id.message) res.status(500).send(id.message)
} else { } else {
res.send(`<script>window.location.href = '${process.env.FRONTEND}/playlist?list=${id}'</script>`) res.redirect(`${process.env.FRONTEND}/playlist?list=${id}`)
} }
} }
@ -33,6 +47,6 @@ exports.searchChannel = async (req, res) => {
if (id.fail) { if (id.fail) {
res.status(500).send(id.message) res.status(500).send(id.message)
} else { } else {
res.send(`<script>window.location.href = '${process.env.FRONTEND}/channel/${id}'</script>`) res.redirect(`${process.env.FRONTEND}/channel/${id}`)
} }
} }

25
controller/transparency.js
View File

@ -1,14 +1,25 @@
const { PrismaClient } = require('@prisma/client') const { PrismaClient } = require('@prisma/client')
const redis = require('../utils/redis.js')
const prisma = new PrismaClient() const prisma = new PrismaClient()
exports.getReports = async (req, res) => { exports.getReports = async (req, res) => {
res.json((await prisma.reports.findMany()).map(r => { let json
return { const cached = await redis.get('transparency')
...r,
details: (r.details).split('<').join('&lt;').split('>').join('&gt;'), if (cached) {
date: (r.date).toISOString().slice(0,10) json = JSON.parse(cached)
} } else {
})) json = (await prisma.reports.findMany()).map(r => {
return {
...r,
details: (r.details).split('<').join('&lt;').split('>').join('&gt;'),
date: (r.date).toISOString().slice(0,10)
}
})
await redis.set('transparency', JSON.stringify(json), 'EX', 3600)
}
res.json(json)
} }
exports.getReport = async (req, res) => { exports.getReport = async (req, res) => {

223
controller/video.js
View File

@ -3,28 +3,37 @@ const prisma = new PrismaClient()
const DOMPurify = require('isomorphic-dompurify') const DOMPurify = require('isomorphic-dompurify')
const metadata = require('../utils/metadata.js') const metadata = require('../utils/metadata.js')
const redis = require('../utils/redis.js')
exports.getVideo = async (req, res) => { exports.getVideo = async (req, res) => {
const info = await prisma.videos.findFirst({ let info
where: { const cached = await redis.get(`video:${req.params.id}`)
id: req.params.id
}, if (cached) {
select: { info = JSON.parse(cached)
title: true, } else {
description: true, info = await prisma.videos.findFirst({
thumbnail: true, where: {
source: true, id: req.params.id
published: true, },
archived: true, select: {
channel: true, title: true,
channelId: true, description: true,
channelAvatar: true, thumbnail: true,
channelVerified: true, source: true,
disabled: true published: true,
} archived: true,
}) channel: true,
channelId: true,
if (!info) return res.json({ error: '404' }) channelAvatar: true,
channelVerified: true,
disabled: true
}
})
if (!info) return res.json({ error: '404' })
await redis.set(`video:${req.params.id}`, JSON.stringify(info), 'EX', 3600)
}
res.json({ res.json({
...info, ...info,
@ -33,11 +42,17 @@ exports.getVideo = async (req, res) => {
} }
exports.getChannel = async (req, res) => { exports.getChannel = async (req, res) => {
const videos = await metadata.getChannelVideos(req.params.id) const cached = await redis.get(`channel:${req.params.id}`)
const channel = await metadata.getChannel(req.params.id) if (cached) return res.json(JSON.parse(cached))
if (!videos || !channel) return res.json({ error: '500' }) const [videos, channel] = await Promise.all([
if (videos.error) return res.json({ error: '404' }) metadata.getChannelVideos(req.params.id),
metadata.getChannel(req.params.id)
])
if (!videos || !channel || videos.error) {
return res.json({ error: '404' });
}
const archived = await prisma.videos.findMany({ const archived = await prisma.videos.findMany({
where: { where: {
@ -52,43 +67,66 @@ exports.getChannel = async (req, res) => {
} }
}) })
var allVideos = [] const processedVideos = videos.map(video => ({
allVideos = allVideos.concat((videos).map(video => { id: video.url.replace('/watch?v=', ''),
return { published: new Date(video.uploaded).toISOString().slice(0, 10),
id: video.url.replace('/watch?v=', ''), ...video
published: (new Date(video.uploaded)).toISOString().slice(0,10), }));
...video
} archived.forEach(v => {
})) const existingVideoIndex = processedVideos.findIndex(video => video.id === v.id);
if (existingVideoIndex !== -1) {
await Promise.all(archived.map(async (v) => { processedVideos[existingVideoIndex] = v;
const allVideo = allVideos.find(o => o.id == v.id)
if (allVideo) {
const index = allVideos.findIndex(o => o.id == v.id)
allVideos[index] = v
} else { } else {
allVideos.push({ processedVideos.push({ ...v, deleted: undefined });
...v,
deleted: undefined
})
} }
})) });
allVideos.sort((a, b) => new Date(b.published) - new Date(a.published)) processedVideos.sort((a, b) => new Date(b.published) - new Date(a.published));
res.json({ const json = {
name: channel.author, name: channel.author,
avatar: channel.authorThumbnails[1].url, avatar: channel.authorThumbnails[1].url,
verified: channel.authorVerified, verified: channel.authorVerified,
videos: allVideos videos: processedVideos
}
await redis.set(`channel:${req.params.id}`, JSON.stringify(json), 'EX', 3600)
res.json(json)
}
exports.getOnlyChannelVideos = async (req, res) => {
const cached = await redis.get(`channelVideos:${req.params.id}`)
if (cached) return res.json(JSON.parse(cached))
const archived = await prisma.videos.findMany({
where: {
channelId: req.params.id
},
select: {
id: true,
title: true,
thumbnail: true,
published: true,
archived: true
},
orderBy: {
published: 'desc'
}
}) })
const json = {
videos: archived
}
await redis.set(`channelVideos:${req.params.id}`, JSON.stringify(json), 'EX', 3600)
res.json(json)
} }
exports.getPlaylist = async (req, res) => { exports.getPlaylist = async (req, res) => {
const cached = await redis.get(`playlist:${req.params.id}`)
if (cached) return res.json(JSON.parse(cached))
const playlist = await metadata.getPlaylistVideos(req.params.id) const playlist = await metadata.getPlaylistVideos(req.params.id)
if (!playlist || playlist.error) return res.json({ error: '404' })
if (!playlist) return res.json({ error: '500' })
if (playlist.error) return res.json({ error: '404' })
const playlistArchived = await prisma.videos.findMany({ const playlistArchived = await prisma.videos.findMany({
where: { where: {
@ -103,59 +141,54 @@ exports.getPlaylist = async (req, res) => {
} }
}) })
var allVideos = [] const allVideos = playlist.relatedStreams.map(video => ({
allVideos = allVideos.concat((playlist.relatedStreams).map(video => { id: video.url.replace('/watch?v=', ''),
return { published: new Date(video.uploaded).toISOString().slice(0, 10),
id: video.url.replace('/watch?v=', ''), ...video
published: (new Date(video.uploaded)).toISOString().slice(0,10), }));
...video
}
}))
await Promise.all(playlistArchived.map(async (v) => { await Promise.all(playlistArchived.map(async (v) => {
const allVideo = allVideos.find(o => o.id == v.id) const allVideo = allVideos.find(o => o.id == v.id);
if (allVideo) { if (allVideo) {
const index = allVideos.findIndex(o => o.id == v.id) const index = allVideos.findIndex(o => o.id == v.id);
allVideos[index] = v allVideos[index] = v;
} else { } else {
const live = await metadata.getVideoMetadata(v.id) const live = await metadata.getVideoMetadata(v.id);
allVideos.push({ allVideos.push({
...v, ...v,
deleted: live.error ? true : false deleted: live.error ? true : false
}) });
} }
})) }));
await Promise.all(allVideos.map(async (v) => { await Promise.all(allVideos.filter(v => !v.archived).map(async (v) => {
if (!v.archived) { const video = await prisma.videos.findFirst({
const video = await prisma.videos.findFirst({ where: {
where: { id: v.id
id: v.id },
}, select: {
select: { id: true,
id: true, title: true,
title: true, thumbnail: true,
thumbnail: true, published: true,
published: true, archived: true
archived: true
}
})
if (video) {
const index = allVideos.findIndex(o => o.id == v.id)
allVideos[index] = video
} }
});
if (video) {
const index = allVideos.findIndex(o => o.id == v.id);
allVideos[index] = video;
} }
})) }));
allVideos.sort((a, b) => new Date(a.published) - new Date(b.published)) allVideos.sort((a, b) => new Date(b.published) - new Date(a.published));
res.json({ const json = {
name: playlist.name, name: playlist.name,
channel: playlist.uploader, channel: playlist.uploader,
url: playlist.uploaderUrl, url: playlist.uploaderUrl,
avatar: playlist.uploaderAvatar, avatar: playlist.uploaderAvatar,
videos: allVideos videos: allVideos
}) }
await redis.set(`playlist:${req.params.id}`, JSON.stringify(json), 'EX', 3600)
res.json(json)
} }

1
index.js
View File

@ -19,6 +19,7 @@ app.use(cors())
app.get('/latest', latestController.getLatest) app.get('/latest', latestController.getLatest)
app.get('/video/:id', videoController.getVideo) app.get('/video/:id', videoController.getVideo)
app.get('/channel/:id', videoController.getChannel) app.get('/channel/:id', videoController.getChannel)
app.get('/channel/:id/videos', videoController.getOnlyChannelVideos)
app.get('/playlist/:id', videoController.getPlaylist) app.get('/playlist/:id', videoController.getPlaylist)
app.get('/search/video', searchController.searchVideo) app.get('/search/video', searchController.searchVideo)

6
utils/ytdlp.js
View File

@ -21,13 +21,13 @@ async function downloadVideo(url, ws) {
}) })
child.on("close", async (code, signal) => { child.on("close", async (code, signal) => {
if (code == 2) { if (code == 0) { // https://itsfoss.com/linux-exit-codes/
reject({ reject({
fail: true fail: false
}) })
} else { } else {
resolve({ resolve({
fail: false fail: true
}) })
} }
}) })