From e6ecd683e4482e89d26348675a7bf028c7e701a3 Mon Sep 17 00:00:00 2001 From: localhost Date: Fri, 29 Mar 2024 18:48:59 +0100 Subject: [PATCH] cobalt integration --- controller/websocket.js | 8 +---- package.json | 2 ++ utils/metadata.js | 18 +++++++++- utils/ytdlp.js | 79 +++++++++++++---------------------------- yarn.lock | 23 ++++++++++++ 5 files changed, 68 insertions(+), 62 deletions(-) diff --git a/controller/websocket.js b/controller/websocket.js index 7f2e997..2dab0d9 100644 --- a/controller/websocket.js +++ b/controller/websocket.js @@ -71,8 +71,6 @@ exports.save = async (ws, req) => { }) async function startDownloading() { - ws.send('INFO - Spawning yt-dlp!') - const download = await ytdlp.downloadVideo(`https://www.youtube.com/watch?v=${id}`, ws, id) if (download.fail) { await redis.del(id) @@ -80,9 +78,7 @@ exports.save = async (ws, req) => { ws.close() } else { const file = fs.readdirSync("videos").find(f => f.includes(id)) - if (file) { - fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`) - + if (file) { ws.send('DATA - Uploading file...') const videoUrl = await upload.uploadVideo(`./videos/${id}.webm`) fs.unlinkSync(`./videos/${id}.webm`) @@ -179,7 +175,6 @@ exports.playlist = async (ws, req) => { const file = fs.readdirSync("./videos").find(f => f.includes(id)) if (file) { try { - fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`) ws.send(`DATA - Downloaded ${video.title}`) ws.send(`DATA - Uploading ${video.title}`) @@ -278,7 +273,6 @@ exports.channel = async (ws, req) => { const file = fs.readdirSync("./videos").find(f => f.includes(id)) if (file) { try { - fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`) ws.send(`DATA - Downloaded ${video.title}`) ws.send(`DATA - Uploading ${video.title}`) diff --git a/package.json b/package.json index d5074b1..5eda145 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "@logtail/node": "^0.4.0", "@logtail/winston": "^0.4.1", "@prisma/client": "4.9.0", + "@tsmx/human-readable": "^2.0.2", "aws-sdk": "2.1128.0", "child_process": "^1.0.2", "cors": "^2.8.5", @@ -18,6 +19,7 @@ "isomorphic-dompurify": "^1.0.0", "node-fetch": "2", "rolling-rate-limiter": "^0.4.2", + "wget-improved": "^3.4.0", "winston": "^3.8.2" }, "devDependencies": { diff --git a/utils/metadata.js b/utils/metadata.js index 7ab5628..51dfe41 100644 --- a/utils/metadata.js +++ b/utils/metadata.js @@ -117,4 +117,20 @@ async function getPlaylistVideos(id) { return json } -module.exports = { getInstance, getVideoMetadata, getChannel, getChannelVideos, getPlaylistVideos } \ No newline at end of file +async function getVideoDownload(url, quality) { + const json = await (await fetch('http://cobalt-api:9000/api/json', { + method: 'POST', + headers: { + 'Accept': 'application/json', + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + 'url': url, + 'vQuality': quality + }) + })).json() + + return json +} + +module.exports = { getInstance, getVideoMetadata, getChannel, getChannelVideos, getPlaylistVideos, getVideoDownload } \ No newline at end of file diff --git a/utils/ytdlp.js b/utils/ytdlp.js index 9da4302..e35278d 100644 --- a/utils/ytdlp.js +++ b/utils/ytdlp.js @@ -1,75 +1,46 @@ -const child_process = require('child_process') +const wget = require('wget-improved') const DOMPurify = require('isomorphic-dompurify') const metadata = require('./metadata.js') +const hr = require('@tsmx/human-readable') async function downloadVideo(url, ws, id) { return new Promise(async (resolve, reject) => { - const args = ['--proxy', 'socks5://gluetun:1080', url] + let quality = '720p' const video = await metadata.getVideoMetadata(id) - if (video.lengthSeconds > 1500) { - const formats = await getFormats(url, ws) - if (!formats.fail && formats.includes('360p')) { - args.push('-f 18') - } - } + if (video.lengthSeconds > 1200) quality = '480p' // 20 minutes + if (video.lengthSeconds > 2100) quality = '360p' // 35 minutes + const downloadJson = await metadata.getVideoDownload(url, quality) - const child = child_process.spawn('../yt-dlp', args, {cwd: 'videos', shell: false}) - // https://github.com/yt-dlp/yt-dlp/blob/cc8d8441524ec3442d7c0d3f8f33f15b66aa06f3/README.md?plain=1#L1500 + let size = '' + const alreadyPrecentages = [] + const download = wget.download(downloadJson.url, `./videos/${id}.webm`) - child.stdout.on("data", data => { - const msg = data.toString().trim() - if (!msg) return - - if (ws) ws.send(`DATA - ${DOMPurify.sanitize(msg)}`) + download.on('start', fileSize => { + size = fileSize + if (ws) ws.send(`DATA - Download has started in ${quality}`) }) - child.stderr.on("data", data => { - const msg = data.toString().trim() - if (!msg) return - - if (ws) ws.send(`DATA - ${DOMPurify.sanitize(msg)}`) + download.on('progress', progress => { + if (alreadyPrecentages.includes((progress*100).toFixed(0))) return + alreadyPrecentages.push((progress*100).toFixed(0)) + + if (ws) ws.send(`DATA - [download] ${(progress*100).toFixed(2)}% of ${hr.fromBytes(size)}`) }) - child.on("close", async (code, signal) => { - if (code == 2 || code == 1) { // https://github.com/yt-dlp/yt-dlp/issues/4262 - reject({ - fail: true - }) - } else { + download.on('error', err => { + if (ws) ws.send(`DATA - ${DOMPurify.sanitize(err)}`) + }) + + download.on('end', output => { + if (output == 'Finished writing to disk') { + ws.send(`DATA - Download has finished`) resolve({ fail: false }) - } - }) - }) -} - -async function getFormats(url, ws) { - return new Promise((resolve, reject) => { - const child = child_process.spawn('../yt-dlp', ['--proxy', 'socks5://gluetun:1080', url, '-F'], {cwd: 'videos', shell: false}) - let outputs = '' - - child.stdout.on("data", data => { - const msg = data.toString().trim() - if (!msg) return - - outputs = outputs + msg - }) - - child.stderr.on("data", data => { - const msg = data.toString().trim() - if (!msg) return - - if (ws) ws.send(`DATA - ${DOMPurify.sanitize(msg)}`) - }) - - child.on("close", async (code, signal) => { - if (code == 2 || code == 1) { // https://github.com/yt-dlp/yt-dlp/issues/4262 + } else { reject({ fail: true }) - } else { - resolve(outputs) } }) }) diff --git a/yarn.lock b/yarn.lock index e2b137a..e3aff2f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -92,6 +92,11 @@ resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-2.0.0.tgz#f544a148d3ab35801c1f633a7441fd87c2e484bf" integrity sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A== +"@tsmx/human-readable@^2.0.2": + version "2.0.2" + resolved "https://registry.yarnpkg.com/@tsmx/human-readable/-/human-readable-2.0.2.tgz#12e65a34120146c44fc18eef3b152e677a5cb5c0" + integrity sha512-KcIboIIHJsD/vVqb9YFRmAnmtZXlA+Qi9Qv0BCbk9TFAEr5hjpKkSUNfrS1LeGMY/UhKZrJldg7x3JNeyh2byQ== + "@types/dompurify@^2.4.0": version "2.4.0" resolved "https://registry.yarnpkg.com/@types/dompurify/-/dompurify-2.4.0.tgz#fd9706392a88e0e0e6d367f3588482d817df0ab9" @@ -862,6 +867,11 @@ minimatch@^3.0.4: dependencies: brace-expansion "^1.1.7" +minimist@1.2.6: + version "1.2.6" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44" + integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q== + ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" @@ -1216,6 +1226,11 @@ triple-beam@^1.3.0: resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9" integrity sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw== +tunnel@0.0.6: + version "0.0.6" + resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c" + integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg== + type-check@~0.3.2: version "0.3.2" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.3.2.tgz#5884cab512cf1d355e3fb784f30804b2b520db72" @@ -1299,6 +1314,14 @@ webidl-conversions@^7.0.0: resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-7.0.0.tgz#256b4e1882be7debbf01d05f0aa2039778ea080a" integrity sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g== +wget-improved@^3.4.0: + version "3.4.0" + resolved "https://registry.yarnpkg.com/wget-improved/-/wget-improved-3.4.0.tgz#da4d2578e46c6ed8532e6d34cbdf8c7344fdd17c" + integrity sha512-mHCdqImHntGzaauaQrfhkcHO0sAOp9Fd/9v5PXwrvHK+nggRWG9en5UH72/WitJFv3d3iFwJSAVMrRaCjW6dAA== + dependencies: + minimist "1.2.6" + tunnel "0.0.6" + whatwg-encoding@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz#e7635f597fd87020858626805a2729fa7698ac53"