cobalt integration

controller/websocket.js

@@ -71,8 +71,6 @@ exports.save = async (ws, req) => {
     })
 
     async function startDownloading() {
-        ws.send('INFO - Spawning yt-dlp!')
-
         const download = await ytdlp.downloadVideo(`https://www.youtube.com/watch?v=${id}`, ws, id)
         if (download.fail) {
             await redis.del(id)
@@ -81,8 +79,6 @@ exports.save = async (ws, req) => {
         } else {
             const file = fs.readdirSync("videos").find(f => f.includes(id))
             if (file) {    
-                fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`)
-    
                 ws.send('DATA - Uploading file...')
                 const videoUrl = await upload.uploadVideo(`./videos/${id}.webm`)
                 fs.unlinkSync(`./videos/${id}.webm`)
@@ -179,7 +175,6 @@ exports.playlist = async (ws, req) => {
                 const file = fs.readdirSync("./videos").find(f => f.includes(id))
                 if (file) {
                     try {
-                        fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`)
                         ws.send(`DATA - Downloaded ${video.title}`)
                         ws.send(`DATA - Uploading ${video.title}`)
 
@@ -278,7 +273,6 @@ exports.channel = async (ws, req) => {
                 const file = fs.readdirSync("./videos").find(f => f.includes(id))
                 if (file) {
                     try {
-                        fs.renameSync(`./videos/${file}`, `./videos/${id}.webm`)
                         ws.send(`DATA - Downloaded ${video.title}`)
                         ws.send(`DATA - Uploading ${video.title}`)
 

package.json

@@ -7,6 +7,7 @@
     "@logtail/node": "^0.4.0",
     "@logtail/winston": "^0.4.1",
     "@prisma/client": "4.9.0",
+    "@tsmx/human-readable": "^2.0.2",
     "aws-sdk": "2.1128.0",
     "child_process": "^1.0.2",
     "cors": "^2.8.5",
@@ -18,6 +19,7 @@
     "isomorphic-dompurify": "^1.0.0",
     "node-fetch": "2",
     "rolling-rate-limiter": "^0.4.2",
+    "wget-improved": "^3.4.0",
     "winston": "^3.8.2"
   },
   "devDependencies": {

utils/metadata.js

@@ -117,4 +117,20 @@ async function getPlaylistVideos(id) {
     return json
 }
 
-module.exports = { getInstance, getVideoMetadata, getChannel, getChannelVideos, getPlaylistVideos }
+async function getVideoDownload(url, quality) {
+    const json = await (await fetch('http://cobalt-api:9000/api/json', {
+        method: 'POST',
+        headers: {
+            'Accept': 'application/json',
+            'Content-Type': 'application/json'
+        },
+        body: JSON.stringify({
+            'url': url,
+            'vQuality': quality
+        })
+    })).json()
+
+    return json
+}
+
+module.exports = { getInstance, getVideoMetadata, getChannel, getChannelVideos, getPlaylistVideos, getVideoDownload }

utils/ytdlp.js

@@ -1,75 +1,46 @@
-const child_process = require('child_process')
+const wget = require('wget-improved')
 const DOMPurify = require('isomorphic-dompurify')
 const metadata = require('./metadata.js')
+const hr = require('@tsmx/human-readable')
 
 async function downloadVideo(url, ws, id) {
     return new Promise(async (resolve, reject) => {
-        const args = ['--proxy', 'socks5://gluetun:1080', url]
+        let quality = '720p'
         const video = await metadata.getVideoMetadata(id)
-        if (video.lengthSeconds > 1500) {
+        if (video.lengthSeconds > 1200) quality = '480p' // 20 minutes
-            const formats = await getFormats(url, ws)
+        if (video.lengthSeconds > 2100) quality = '360p' // 35 minutes
-            if (!formats.fail && formats.includes('360p')) {
+        const downloadJson = await metadata.getVideoDownload(url, quality)
-                args.push('-f 18')
-            }
-        }
 
-        const child = child_process.spawn('../yt-dlp', args, {cwd: 'videos', shell: false})
+        let size = ''
-        // https://github.com/yt-dlp/yt-dlp/blob/cc8d8441524ec3442d7c0d3f8f33f15b66aa06f3/README.md?plain=1#L1500
+        const alreadyPrecentages = []
+        const download = wget.download(downloadJson.url, `./videos/${id}.webm`)
         
-        child.stdout.on("data", data => {
+        download.on('start', fileSize => {
-            const msg = data.toString().trim()
+            size = fileSize
-            if (!msg) return 
+            if (ws) ws.send(`DATA - Download has started in ${quality}`)
-    
-            if (ws) ws.send(`DATA - ${DOMPurify.sanitize(msg)}`)
         })
 
-        child.stderr.on("data", data => {
+        download.on('progress', progress => {
-            const msg = data.toString().trim()
+            if (alreadyPrecentages.includes((progress*100).toFixed(0))) return 
-            if (!msg) return 
+            alreadyPrecentages.push((progress*100).toFixed(0))
             
-            if (ws) ws.send(`DATA - ${DOMPurify.sanitize(msg)}`)
+            if (ws) ws.send(`DATA - [download] ${(progress*100).toFixed(2)}% of ${hr.fromBytes(size)}`)
         })
 
-        child.on("close", async (code, signal) => {
+        download.on('error', err => {
-            if (code == 2 || code == 1) { // https://github.com/yt-dlp/yt-dlp/issues/4262
+            if (ws) ws.send(`DATA - ${DOMPurify.sanitize(err)}`)
-                reject({
+        })
-                    fail: true
+
-                })
+        download.on('end', output => {
-            } else {
+            if (output == 'Finished writing to disk') {
+                ws.send(`DATA - Download has finished`)
                 resolve({
                     fail: false
                 })
-            }
+            } else {
-        })
-    })
-}
-
-async function getFormats(url, ws) {
-    return new Promise((resolve, reject) => {
-        const child = child_process.spawn('../yt-dlp', ['--proxy', 'socks5://gluetun:1080', url, '-F'], {cwd: 'videos', shell: false})
-        let outputs = ''
-
-        child.stdout.on("data", data => {
-            const msg = data.toString().trim()
-            if (!msg) return 
-            
-            outputs = outputs + msg
-        })
-
-        child.stderr.on("data", data => {
-            const msg = data.toString().trim()
-            if (!msg) return 
-    
-            if (ws) ws.send(`DATA - ${DOMPurify.sanitize(msg)}`)
-        })
-
-        child.on("close", async (code, signal) => {
-            if (code == 2 || code == 1) { // https://github.com/yt-dlp/yt-dlp/issues/4262
                 reject({
                     fail: true
                 })
-            } else {
-                resolve(outputs)
             }
         })
     })

yarn.lock

@@ -92,6 +92,11 @@
   resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-2.0.0.tgz#f544a148d3ab35801c1f633a7441fd87c2e484bf"
   integrity sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==
 
+"@tsmx/human-readable@^2.0.2":
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/@tsmx/human-readable/-/human-readable-2.0.2.tgz#12e65a34120146c44fc18eef3b152e677a5cb5c0"
+  integrity sha512-KcIboIIHJsD/vVqb9YFRmAnmtZXlA+Qi9Qv0BCbk9TFAEr5hjpKkSUNfrS1LeGMY/UhKZrJldg7x3JNeyh2byQ==
+
 "@types/dompurify@^2.4.0":
   version "2.4.0"
   resolved "https://registry.yarnpkg.com/@types/dompurify/-/dompurify-2.4.0.tgz#fd9706392a88e0e0e6d367f3588482d817df0ab9"
@@ -862,6 +867,11 @@ minimatch@^3.0.4:
   dependencies:
     brace-expansion "^1.1.7"
 
+minimist@1.2.6:
+  version "1.2.6"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44"
+  integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==
+
 ms@2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
@@ -1216,6 +1226,11 @@ triple-beam@^1.3.0:
   resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9"
   integrity sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==
 
+tunnel@0.0.6:
+  version "0.0.6"
+  resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c"
+  integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==
+
 type-check@~0.3.2:
   version "0.3.2"
   resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.3.2.tgz#5884cab512cf1d355e3fb784f30804b2b520db72"
@@ -1299,6 +1314,14 @@ webidl-conversions@^7.0.0:
   resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-7.0.0.tgz#256b4e1882be7debbf01d05f0aa2039778ea080a"
   integrity sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==
 
+wget-improved@^3.4.0:
+  version "3.4.0"
+  resolved "https://registry.yarnpkg.com/wget-improved/-/wget-improved-3.4.0.tgz#da4d2578e46c6ed8532e6d34cbdf8c7344fdd17c"
+  integrity sha512-mHCdqImHntGzaauaQrfhkcHO0sAOp9Fd/9v5PXwrvHK+nggRWG9en5UH72/WitJFv3d3iFwJSAVMrRaCjW6dAA==
+  dependencies:
+    minimist "1.2.6"
+    tunnel "0.0.6"
+
 whatwg-encoding@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz#e7635f597fd87020858626805a2729fa7698ac53"