preservetube
/
backend
mirror of https://github.com/preservetube/backend.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fuck my life. channels migrated to the youtube direct api

This commit is contained in:
localhost 2024-03-30 20:05:11 +01:00
parent 2bc8257a39
commit d059a4acad
5 changed files with 62 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
controller/video.js
View File

@ -2,6 +2,7 @@ const { PrismaClient } = require('@prisma/client')
const prisma = new PrismaClient()
const DOMPurify = require('isomorphic-dompurify')
const rtm = require('readable-to-ms')
const metadata = require('../utils/metadata.js')
const redis = require('../utils/redis.js')
@ -68,12 +69,16 @@ exports.getChannel = async (req, res) => {
}
})
const processedVideos = videos.map(video => ({
id: video.url.replace('/watch?v=', ''),
published: new Date(video.uploaded).toISOString().slice(0, 10),
...video
}));
const processedVideos = videos.map(video => {
const date = !isNaN(new Date(video.published.text).getTime()) ? new Date(video.published.text) : new Date((new Date()).getTime() - rtm(video.published.text).ms); // life is great.
return {
id: video.id,
title: video.title.text,
thumbnail: video.thumbnails[0].url,
published: new Date(date).toISOString().slice(0, 10)
}
});
archived.forEach(v => {
const existingVideoIndex = processedVideos.findIndex(video => video.id === v.id);
if (existingVideoIndex !== -1) {

47
controller/websocket.js
View File

@ -1,6 +1,7 @@
const fs = require('node:fs')
const crypto = require('node:crypto')
const { RedisRateLimiter } = require('rolling-rate-limiter')
const rtm = require('readable-to-ms')
const upload = require('../utils/upload.js')
const ytdlp = require('../utils/ytdlp.js')
@ -233,64 +234,62 @@ exports.channel = async (ws, req) => {
async function startDownloading() {
const videos = await metadata.getChannelVideos(channelId)
for (video of videos.slice(0, 5)) {
for (const video of videos.slice(0, 5)) {
if (ws.readyState !== ws.OPEN) {
return logger.info({ message: `Stopped downloading ${channelId}, websocket is closed` })
}
const id = video.url.match(/[?&]v=([^&]+)/)[1]
const already = await prisma.videos.findFirst({
where: {
id: id
id: video.id
}
})
if (already) {
ws.send(`DATA - Already downloaded ${video.title}`)
ws.send(`DATA - Already downloaded ${video.title.text}`)
continue
}
if (await redis.get(id)) {
ws.send(`DATA - Someone is already downloading ${video.title}, skipping.`)
if (await redis.get(video.id)) {
ws.send(`DATA - Someone is already downloading ${video.title.text}, skipping.`)
continue
}
if (await redis.get(`blacklist:${id}`)) {
ws.send(`DATA - ${video.title} is blacklisted from downloading, skipping`)
if (await redis.get(`blacklist:${video.id}`)) {
ws.send(`DATA - ${video.title.text} is blacklisted from downloading, skipping`)
continue
}
ws.send(`INFO - Downloading ${video.title}<br><br>`)
await redis.set(id, 'downloading')
ws.send(`INFO - Downloading ${video.title.text}<br><br>`)
await redis.set(video.id, 'downloading')
const download = await ytdlp.downloadVideo('https://www.youtube.com' + video.url, ws, id)
const download = await ytdlp.downloadVideo(`https://www.youtube.com/watch?v=${video.id}`, ws, video.id)
if (download.fail) {
ws.send(`DATA - ${download.message}`)
await redis.del(id)
await redis.del(video.id)
continue
} else {
const file = fs.readdirSync("./videos").find(f => f.includes(id))
const file = fs.readdirSync("./videos").find(f => f.includes(video.id))
if (file) {
try {
ws.send(`DATA - Downloaded ${video.title}`)
ws.send(`DATA - Uploading ${video.title}`)
ws.send(`DATA - Downloaded ${video.title.text}`)
ws.send(`DATA - Uploading ${video.title.text}`)
const videoUrl = await upload.uploadVideo(`./videos/${id}.mp4`)
ws.send(`DATA - Uploaded ${video.title}`)
fs.unlinkSync(`./videos/${id}.mp4`)
const videoUrl = await upload.uploadVideo(`./videos/${video.id}.mp4`)
ws.send(`DATA - Uploaded ${video.title.text}`)
fs.unlinkSync(`./videos/${video.id}.mp4`)
await websocket.createDatabaseVideo(id, videoUrl)
ws.send(`DATA - Created video page for ${video.title}`)
await websocket.createDatabaseVideo(video.id, videoUrl)
ws.send(`DATA - Created video page for ${video.title.text}`)
} catch (e) {
ws.send(`DATA - Failed downloading video ${video.title}. Going to next video`)
ws.send(`DATA - Failed downloading video ${video.title.text}. Going to next video`)
logger.error(e)
}
} else {
ws.send(`DATA - Failed to find file for ${video.title}. Going to next video`)
ws.send(`DATA - Failed to find file for ${video.title.text}. Going to next video`)
}
await redis.del(id)
await redis.del(video.id)
}
}

1
package.json
View File

@ -18,6 +18,7 @@
"ioredis": "^5.3.1",
"isomorphic-dompurify": "^1.0.0",
"node-fetch": "2",
"readable-to-ms": "^1.0.3",
"rolling-rate-limiter": "^0.4.2",
"wget-improved": "^3.4.0",
"winston": "^3.8.2",

65
utils/metadata.js
View File

@ -1,31 +1,15 @@
const { Innertube } = require('youtubei.js');
const fetch = require('node-fetch')
const https = require('https')
const maxRetries = 5
const platforms = ['WEB', 'ANDROID', 'iOS']
const cobalt = ['http://cobalt-api:9000', 'https://co.wuk.sh', 'http://cobalt-api:9000']
const ignoreSsl = new https.Agent({
rejectUnauthorized: false,
})
async function getInstance() {
const instances = await (await fetch('https://api.invidious.io/instances.json?pretty=1', {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; PreserveTube/0.0; +https://preservetube.com)'
}
})).json()
const sorted = instances.filter(o => o[1].type == 'https' && o[0] != 'invidious.io.lol' && o[0] != 'invidious.0011.lt')
return `https://${sorted[Math.floor(Math.random() * sorted.length)][0]}`
}
async function getPipedInstance() {
const instances = await (await fetch('https://piped-instances.kavin.rocks/', {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; PreserveTube/0.0; +https://preservetube.com)'
},
agent: ignoreSsl
}
})).json()
return (instances[Math.floor(Math.random() * instances.length)]).api_url
}
@ -85,34 +69,29 @@ async function getChannel(id) {
async function getChannelVideos(id) {
return new Promise(async (resolve, reject) => {
try {
const videos = []
const instance = await getPipedInstance()
const json = await (await fetch(`${instance}/channel/${id}`, {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; PreserveTube/0.0; +https://preservetube.com)'
}
})).json()
videos.push(...json.relatedStreams)
if (json.nextpage) await getNextPage(json.nextpage)
else resolve(videos)
async function getNextPage(payload) {
const page = await (await fetch(`${instance}/nextpage/channel/${id}?nextpage=${encodeURIComponent(payload)}`, {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; PreserveTube/0.0; +https://preservetube.com)'
}
})).json()
videos.push(...page.relatedStreams)
if (videos.length >= 60) return resolve(videos)
if (page.nextpage) await getNextPage(page.nextpage)
else return resolve(videos)
const videos = [];
const yt = await Innertube.create();
const channel = await yt.getChannel(id);
let json = await channel.getVideos();
videos.push(...json.videos);
while (json.has_continuation && videos.length < 60) {
json = await getNextPage(json);
videos.push(...json.videos);
}
resolve(videos);
} catch (e) {
resolve(false)
resolve(false);
}
})
});
async function getNextPage(json) {
const page = await json.getContinuation();
return page;
}
}
async function getPlaylistVideos(id) {
@ -153,4 +132,4 @@ async function getVideoDownload(url, quality) {
return json
}
module.exports = { getInstance, getVideoMetadata, getChannel, getChannelVideos, getPlaylistVideos, getVideoDownload }
module.exports = { getVideoMetadata, getChannel, getChannelVideos, getPlaylistVideos, getVideoDownload }

5
yarn.lock
View File

@ -1067,6 +1067,11 @@ readable-stream@^3.4.0, readable-stream@^3.6.0:
string_decoder "^1.1.1"
util-deprecate "^1.0.1"
readable-to-ms@^1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/readable-to-ms/-/readable-to-ms-1.0.3.tgz#7a1bc1cdf15a13e0808d488c160ea723fd04e60f"
integrity sha512-xxxnoflc1CT6st5azNw5u8m739XkrqrkYgLGTe9noe2OG5ohxz3viFQoOSXN2qdckf79da353uBPHQrsjPuYdQ==
redis-errors@^1.0.0, redis-errors@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad"