preservetube
/
backend
mirror of https://github.com/preservetube/backend.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

don't rely on apis for "important" stuff

This commit is contained in:
localhost 2024-03-30 11:11:49 +01:00
parent b805566045
commit 593c155fdf
7 changed files with 78 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
controller/video.js
View File

@ -51,7 +51,7 @@ exports.getChannel = async (req, res) => {
metadata.getChannel(req.params.id)
])
if (!videos || !channel || videos.error) {
if (!videos || !channel || videos.error || channel.error) {
return res.json({ error: '404' });
}
@ -86,9 +86,9 @@ exports.getChannel = async (req, res) => {
processedVideos.sort((a, b) => new Date(b.published) - new Date(a.published));
const json = {
name: channel.author,
avatar: channel.authorThumbnails[1].url,
verified: channel.authorVerified,
name: channel.metadata.title,
avatar: channel.metadata.avatar[0].url,
verified: channel.header.author.is_verified,
videos: processedVideos
}
await redis.set(`channel:${req.params.id}`, JSON.stringify(json), 'EX', 3600)

1
index.js
View File

@ -36,6 +36,7 @@ app.get('/autodownload', websocketController.addAutodownload)
process.on('uncaughtException', err => {
logger.error(err)
console.log(err)
})
app.listen(1337, () => {

3
package.json
View File

@ -20,7 +20,8 @@
"node-fetch": "2",
"rolling-rate-limiter": "^0.4.2",
"wget-improved": "^3.4.0",
"winston": "^3.8.2"
"winston": "^3.8.2",
"youtubei.js": "^9.1.0"
},
"devDependencies": {
"prisma": "4.9.0"

36
utils/metadata.js
View File

@ -1,3 +1,4 @@
const { Innertube } = require('youtubei.js');
const fetch = require('node-fetch')
const https = require('https')
const maxRetries = 5
@ -29,19 +30,13 @@ async function getPipedInstance() {
async function getVideoMetadata(id) {
for (let retries = 0; retries < maxRetries; retries++) {
try {
const instance = await getInstance()
const response = await fetch(`${instance}/api/v1/videos/${id}?fields=videoId,title,descriptionHtml,videoThumbnails,published,authorId,lengthSeconds,error&pretty=1`, {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; PreserveTube/0.0; +https://preservetube.com)'
}
})
const yt = await Innertube.create();
const info = await yt.getInfo(id, 'WEB');
if (response.ok) {
const json = await response.json()
return json
} else {
continue
}
if (!info) return { error: 'ErrorCantConnectToServiceAPI' }; // stolen from cobalt :)
if (info.playability_status.status !== 'OK') return { error: 'ErrorYTUnavailable' };
if (info.basic_info.is_live) return { error: 'ErrorLiveVideo' };
return info
} catch (error) {
continue
}
@ -53,19 +48,10 @@ async function getVideoMetadata(id) {
async function getChannel(id) {
for (let retries = 0; retries < maxRetries; retries++) {
try {
const instance = await getInstance()
const response = await fetch(`${instance}/api/v1/channels/${id}?pretty=1`, {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; PreserveTube/0.0; +https://preservetube.com)'
}
})
if (response.ok) {
const json = await response.json()
return json
} else {
continue
}
const yt = await Innertube.create();
const info = await yt.getChannel(id, 'WEB');
if (!info) return { error: 'ErrorCantConnectToServiceAPI' }; // stolen from cobalt :)
return info
} catch (error) {
continue
}

21
utils/websocket.js
View File

@ -6,24 +6,27 @@ const upload = require('./upload.js')
async function createDatabaseVideo(id, videoUrl, playlistId) {
const data = await metadata.getVideoMetadata(id)
const channelData = await metadata.getChannel(data.authorId)
const channelData = await metadata.getChannel(data.basic_info.channel_id)
const uploaderAvatar = await upload.uploadImage(data.authorId, channelData.authorThumbnails[1].url)
const thumbnailUrl = await upload.uploadImage(id, data.videoThumbnails[0].url)
if (data.error) return data
if (channelData.error) return channelData
const uploaderAvatar = await upload.uploadImage(data.basic_info.channel_id, channelData.metadata.thumbnail[0].url)
const thumbnailUrl = await upload.uploadImage(id, data.basic_info.thumbnail[0].url)
await prisma.videos.create({
data: {
id: id,
title: data.title,
description: (data.descriptionHtml).replaceAll('\n', '<br>'),
title: data.basic_info.title,
description: (data.basic_info.short_description).replaceAll('\n', '<br>'),
thumbnail: thumbnailUrl,
source: videoUrl,
published: (new Date(data.published*1000)).toISOString().slice(0,10),
published: (new Date(data.primary_info.published.text)).toISOString().slice(0,10),
archived: (new Date()).toISOString().slice(0,10),
channel: channelData.author,
channelId: channelData.authorId,
channel: channelData.metadata.title,
channelId: channelData.metadata.external_id,
channelAvatar: uploaderAvatar,
channelVerified: channelData.authorVerified,
channelVerified: channelData.header.author.is_verified,
playlist: playlistId
}
})

11
utils/ytdlp.js
View File

@ -7,8 +7,15 @@ async function downloadVideo(url, ws, id) {
return new Promise(async (resolve, reject) => {
let quality = '720p'
const video = await metadata.getVideoMetadata(id)
if (video.lengthSeconds > 1200) quality = '480p' // 20 minutes
if (video.lengthSeconds > 2100) quality = '360p' // 35 minutes
if (video.error) {
resolve({
message: `Failed to request Youtube with error ${video.error}. Please retry...`,
fail: true
})
}
if (video.basic_info.duration > 1200) quality = '480p' // 20 minutes
if (video.basic_info.duration > 2100) quality = '360p' // 35 minutes
const downloadJson = await metadata.getVideoDownload(url, quality)
if (downloadJson.status == 'error') {
resolve({

38
yarn.lock
View File

@ -16,6 +16,11 @@
enabled "2.0.x"
kuler "^2.0.0"
"@fastify/busboy@^2.0.0":
version "2.1.1"
resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d"
integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==
"@ioredis/commands@^1.1.1":
version "1.2.0"
resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11"
@ -150,6 +155,11 @@ acorn@^8.1.0, acorn@^8.8.1:
resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.2.tgz#1b2f25db02af965399b9776b0c2c391276d37c4a"
integrity sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw==
acorn@^8.8.0:
version "8.11.3"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.11.3.tgz#71e0b14e13a4ec160724b38fb7b0f233b1b81d7a"
integrity sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==
agent-base@6:
version "6.0.2"
resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77"
@ -736,6 +746,13 @@ isomorphic-dompurify@^1.0.0:
dompurify "^3.0.0"
jsdom "^21.1.0"
jintr@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/jintr/-/jintr-1.1.0.tgz#223a3b07f5e03d410cec6e715c537c8ad1e714c3"
integrity sha512-Tu9wk3BpN2v+kb8yT6YBtue+/nbjeLFv4vvVC4PJ7oCidHKbifWhvORrAbQfxVIQZG+67am/mDagpiGSVtvrZg==
dependencies:
acorn "^8.8.0"
jmespath@0.16.0:
version "0.16.0"
resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.16.0.tgz#b15b0a85dfd4d930d43e69ed605943c802785076"
@ -1226,6 +1243,11 @@ triple-beam@^1.3.0:
resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9"
integrity sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==
tslib@^2.5.0:
version "2.6.2"
resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae"
integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==
tunnel@0.0.6:
version "0.0.6"
resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c"
@ -1246,6 +1268,13 @@ type-is@~1.6.18:
media-typer "0.3.0"
mime-types "~2.1.24"
undici@^5.19.1:
version "5.28.3"
resolved "https://registry.yarnpkg.com/undici/-/undici-5.28.3.tgz#a731e0eff2c3fcfd41c1169a869062be222d1e5b"
integrity sha512-3ItfzbrhDlINjaP0duwnNsKpDQk3acHI3gVJ1z4fmwMK31k5G9OVIAMLSIaP6w4FaGkaAkN6zaQO9LUvZ1t7VA==
dependencies:
"@fastify/busboy" "^2.0.0"
universalify@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.2.0.tgz#6451760566fa857534745ab1dde952d1b1761be0"
@ -1413,3 +1442,12 @@ xmlchars@^2.2.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"
integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==
youtubei.js@^9.1.0:
version "9.1.0"
resolved "https://registry.yarnpkg.com/youtubei.js/-/youtubei.js-9.1.0.tgz#bcf154c9fa21d3c8c1d00a5e10360d0a065c660e"
integrity sha512-C5GBJ4LgnS6vGAUkdIdQNOFFb5EZ1p3xBvUELNXmIG3Idr6vxWrKNBNy8ClZT3SuDVXaAJqDgF9b5jvY8lNKcg==
dependencies:
jintr "^1.1.0"
tslib "^2.5.0"
undici "^5.19.1"