diff --git a/controller/video.js b/controller/video.js
index f1c29cb..863c8a5 100644
--- a/controller/video.js
+++ b/controller/video.js
@@ -2,6 +2,7 @@ const { PrismaClient } = require('@prisma/client')
const prisma = new PrismaClient()
const DOMPurify = require('isomorphic-dompurify')
+const rtm = require('readable-to-ms')
const metadata = require('../utils/metadata.js')
const redis = require('../utils/redis.js')
@@ -68,12 +69,16 @@ exports.getChannel = async (req, res) => {
}
})
- const processedVideos = videos.map(video => ({
- id: video.url.replace('/watch?v=', ''),
- published: new Date(video.uploaded).toISOString().slice(0, 10),
- ...video
- }));
-
+ const processedVideos = videos.map(video => {
+ const date = !isNaN(new Date(video.published.text).getTime()) ? new Date(video.published.text) : new Date((new Date()).getTime() - rtm(video.published.text).ms); // life is great.
+ return {
+ id: video.id,
+ title: video.title.text,
+ thumbnail: video.thumbnails[0].url,
+ published: new Date(date).toISOString().slice(0, 10)
+ }
+ });
+
archived.forEach(v => {
const existingVideoIndex = processedVideos.findIndex(video => video.id === v.id);
if (existingVideoIndex !== -1) {
diff --git a/controller/websocket.js b/controller/websocket.js
index 7464c3c..4ba7ebb 100644
--- a/controller/websocket.js
+++ b/controller/websocket.js
@@ -1,6 +1,7 @@
const fs = require('node:fs')
const crypto = require('node:crypto')
const { RedisRateLimiter } = require('rolling-rate-limiter')
+const rtm = require('readable-to-ms')
const upload = require('../utils/upload.js')
const ytdlp = require('../utils/ytdlp.js')
@@ -233,64 +234,62 @@ exports.channel = async (ws, req) => {
async function startDownloading() {
const videos = await metadata.getChannelVideos(channelId)
- for (video of videos.slice(0, 5)) {
+ for (const video of videos.slice(0, 5)) {
if (ws.readyState !== ws.OPEN) {
return logger.info({ message: `Stopped downloading ${channelId}, websocket is closed` })
}
- const id = video.url.match(/[?&]v=([^&]+)/)[1]
-
const already = await prisma.videos.findFirst({
where: {
- id: id
+ id: video.id
}
})
if (already) {
- ws.send(`DATA - Already downloaded ${video.title}`)
+ ws.send(`DATA - Already downloaded ${video.title.text}`)
continue
}
- if (await redis.get(id)) {
- ws.send(`DATA - Someone is already downloading ${video.title}, skipping.`)
+ if (await redis.get(video.id)) {
+ ws.send(`DATA - Someone is already downloading ${video.title.text}, skipping.`)
continue
}
- if (await redis.get(`blacklist:${id}`)) {
- ws.send(`DATA - ${video.title} is blacklisted from downloading, skipping`)
+ if (await redis.get(`blacklist:${video.id}`)) {
+ ws.send(`DATA - ${video.title.text} is blacklisted from downloading, skipping`)
continue
}
- ws.send(`INFO - Downloading ${video.title}
`)
- await redis.set(id, 'downloading')
+ ws.send(`INFO - Downloading ${video.title.text}
`)
+ await redis.set(video.id, 'downloading')
- const download = await ytdlp.downloadVideo('https://www.youtube.com' + video.url, ws, id)
+ const download = await ytdlp.downloadVideo(`https://www.youtube.com/watch?v=${video.id}`, ws, video.id)
if (download.fail) {
ws.send(`DATA - ${download.message}`)
- await redis.del(id)
+ await redis.del(video.id)
continue
} else {
- const file = fs.readdirSync("./videos").find(f => f.includes(id))
+ const file = fs.readdirSync("./videos").find(f => f.includes(video.id))
if (file) {
try {
- ws.send(`DATA - Downloaded ${video.title}`)
- ws.send(`DATA - Uploading ${video.title}`)
+ ws.send(`DATA - Downloaded ${video.title.text}`)
+ ws.send(`DATA - Uploading ${video.title.text}`)
- const videoUrl = await upload.uploadVideo(`./videos/${id}.mp4`)
- ws.send(`DATA - Uploaded ${video.title}`)
- fs.unlinkSync(`./videos/${id}.mp4`)
+ const videoUrl = await upload.uploadVideo(`./videos/${video.id}.mp4`)
+ ws.send(`DATA - Uploaded ${video.title.text}`)
+ fs.unlinkSync(`./videos/${video.id}.mp4`)
- await websocket.createDatabaseVideo(id, videoUrl)
- ws.send(`DATA - Created video page for ${video.title}`)
+ await websocket.createDatabaseVideo(video.id, videoUrl)
+ ws.send(`DATA - Created video page for ${video.title.text}`)
} catch (e) {
- ws.send(`DATA - Failed downloading video ${video.title}. Going to next video`)
+ ws.send(`DATA - Failed downloading video ${video.title.text}. Going to next video`)
logger.error(e)
}
} else {
- ws.send(`DATA - Failed to find file for ${video.title}. Going to next video`)
+ ws.send(`DATA - Failed to find file for ${video.title.text}. Going to next video`)
}
- await redis.del(id)
+ await redis.del(video.id)
}
}
diff --git a/package.json b/package.json
index 05832d2..39883e4 100644
--- a/package.json
+++ b/package.json
@@ -18,6 +18,7 @@
"ioredis": "^5.3.1",
"isomorphic-dompurify": "^1.0.0",
"node-fetch": "2",
+ "readable-to-ms": "^1.0.3",
"rolling-rate-limiter": "^0.4.2",
"wget-improved": "^3.4.0",
"winston": "^3.8.2",
diff --git a/utils/metadata.js b/utils/metadata.js
index 9d6ab97..8be8c35 100644
--- a/utils/metadata.js
+++ b/utils/metadata.js
@@ -1,31 +1,15 @@
const { Innertube } = require('youtubei.js');
const fetch = require('node-fetch')
-const https = require('https')
const maxRetries = 5
const platforms = ['WEB', 'ANDROID', 'iOS']
const cobalt = ['http://cobalt-api:9000', 'https://co.wuk.sh', 'http://cobalt-api:9000']
-const ignoreSsl = new https.Agent({
- rejectUnauthorized: false,
-})
-
-async function getInstance() {
- const instances = await (await fetch('https://api.invidious.io/instances.json?pretty=1', {
- headers: {
- 'User-Agent': 'Mozilla/5.0 (compatible; PreserveTube/0.0; +https://preservetube.com)'
- }
- })).json()
- const sorted = instances.filter(o => o[1].type == 'https' && o[0] != 'invidious.io.lol' && o[0] != 'invidious.0011.lt')
- return `https://${sorted[Math.floor(Math.random() * sorted.length)][0]}`
-}
-
async function getPipedInstance() {
const instances = await (await fetch('https://piped-instances.kavin.rocks/', {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; PreserveTube/0.0; +https://preservetube.com)'
- },
- agent: ignoreSsl
+ }
})).json()
return (instances[Math.floor(Math.random() * instances.length)]).api_url
}
@@ -85,34 +69,29 @@ async function getChannel(id) {
async function getChannelVideos(id) {
return new Promise(async (resolve, reject) => {
try {
- const videos = []
- const instance = await getPipedInstance()
- const json = await (await fetch(`${instance}/channel/${id}`, {
- headers: {
- 'User-Agent': 'Mozilla/5.0 (compatible; PreserveTube/0.0; +https://preservetube.com)'
- }
- })).json()
- videos.push(...json.relatedStreams)
- if (json.nextpage) await getNextPage(json.nextpage)
- else resolve(videos)
-
- async function getNextPage(payload) {
- const page = await (await fetch(`${instance}/nextpage/channel/${id}?nextpage=${encodeURIComponent(payload)}`, {
- headers: {
- 'User-Agent': 'Mozilla/5.0 (compatible; PreserveTube/0.0; +https://preservetube.com)'
- }
- })).json()
- videos.push(...page.relatedStreams)
-
- if (videos.length >= 60) return resolve(videos)
- if (page.nextpage) await getNextPage(page.nextpage)
- else return resolve(videos)
+ const videos = [];
+ const yt = await Innertube.create();
+ const channel = await yt.getChannel(id);
+ let json = await channel.getVideos();
+
+ videos.push(...json.videos);
+
+ while (json.has_continuation && videos.length < 60) {
+ json = await getNextPage(json);
+ videos.push(...json.videos);
}
-
+
+ resolve(videos);
+
} catch (e) {
- resolve(false)
+ resolve(false);
}
- })
+ });
+
+ async function getNextPage(json) {
+ const page = await json.getContinuation();
+ return page;
+ }
}
async function getPlaylistVideos(id) {
@@ -153,4 +132,4 @@ async function getVideoDownload(url, quality) {
return json
}
-module.exports = { getInstance, getVideoMetadata, getChannel, getChannelVideos, getPlaylistVideos, getVideoDownload }
\ No newline at end of file
+module.exports = { getVideoMetadata, getChannel, getChannelVideos, getPlaylistVideos, getVideoDownload }
\ No newline at end of file
diff --git a/yarn.lock b/yarn.lock
index ba52f49..0134c3e 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1067,6 +1067,11 @@ readable-stream@^3.4.0, readable-stream@^3.6.0:
string_decoder "^1.1.1"
util-deprecate "^1.0.1"
+readable-to-ms@^1.0.3:
+ version "1.0.3"
+ resolved "https://registry.yarnpkg.com/readable-to-ms/-/readable-to-ms-1.0.3.tgz#7a1bc1cdf15a13e0808d488c160ea723fd04e60f"
+ integrity sha512-xxxnoflc1CT6st5azNw5u8m739XkrqrkYgLGTe9noe2OG5ohxz3viFQoOSXN2qdckf79da353uBPHQrsjPuYdQ==
+
redis-errors@^1.0.0, redis-errors@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad"