diff --git a/index.ts b/index.ts index cac38a9..cb95896 100644 --- a/index.ts +++ b/index.ts @@ -12,6 +12,7 @@ import { } from './utils/sabr-stream-factory.js'; import type { SabrPlaybackOptions } from 'googlevideo/sabr-stream'; import { getVideoStreams, downloadStream, getInfo } from './utils/companion'; +import { createInnertubePool } from './utils/innertube-pool.js'; const ffmpeg = require('fluent-ffmpeg') @@ -20,7 +21,7 @@ require('express-ws')(app) ffmpeg.setFfmpegPath('/usr/local/bin/ffmpeg') const maxRetries = 5 -const platforms = ['IOS', 'ANDROID', 'YTSTUDIO_ANDROID', 'YTMUSIC_ANDROID'] +const innertubePool = createInnertubePool(parseInt(Bun.env.INNERTUBE_POOL_SIZE || '4')) app.get('/health', async (req, res) => { try { @@ -69,56 +70,51 @@ app.get('/video/:id', async (req, res) => { }) app.get('/channel/:id', async (req, res) => { - let error = '' + try { + const info = await innertubePool.use(async (innertube) => await innertube.getChannel(req.params.id), maxRetries) - for (let retries = 0; retries < maxRetries; retries++) { - try { - const yt = await Innertube.create(); - const info = await yt.getChannel(req.params.id); - - if (!info) { - error = 'ErrorCantConnectToServiceAPI' - continue; - } - return res.json(info) - } catch (error) { - continue + if (!info) { + return res.json({ error: 'ErrorCantConnectToServiceAPI' }) } - } - res.json({ error: error || 'ErrorUnknown' }) + return res.json(info) + } catch { + return res.json({ error: 'ErrorUnknown' }) + } }) app.get('/videos/:id', async (req, res) => { try { - let videos: any[] = []; + const videos = await innertubePool.use(async (innertube) => { + for (let attempt = 0; attempt < 3; attempt++) { + const videos: any[] = [] + const channel = await innertube.getChannel(req.params.id) - for (let attempt = 0; attempt < 3; attempt++) { - videos = []; + let json = await channel.getVideos() + videos.push(...json.videos) - const yt = await Innertube.create(); - const channel = await yt.getChannel(req.params.id); + while (json.has_continuation && videos.length < 60) { + json = await getNextPage(json) + videos.push(...json.videos) + } - let json = await channel.getVideos(); - videos.push(...json.videos); - - while (json.has_continuation && videos.length < 60) { - json = await getNextPage(json); - videos.push(...json.videos); + if (videos.length) { + return videos + } } - if (videos.length) break; - } + return [] + }, maxRetries) - return res.json(videos); + return res.json(videos) } catch (e: any) { - console.log(e); + console.log(e) if (e.message.includes('Tab "videos" not found')) { - return res.json([]); + return res.json([]) } - return res.json(false); + return res.json(false) } async function getNextPage(json: any) { @@ -206,30 +202,39 @@ app.ws('/download/:id', async (ws, req) => { audioQuality: 'AUDIO_QUALITY_LOW', enabledTrackTypes: EnabledTrackTypes.VIDEO_AND_AUDIO }; - const { streamResults } = await createSabrStream(req.params.id, streamOptions); - const { videoStream, audioStream, selectedFormats } = streamResults; + const lease = await innertubePool.acquire() - const videoSizeTotal = (selectedFormats.audioFormat.contentLength || 0) - + (selectedFormats.videoFormat.contentLength || 0) + try { + const { streamResults } = await createSabrStream(req.params.id, streamOptions, lease.innertube) + const { videoStream, audioStream, selectedFormats } = streamResults - if (videoSizeTotal > (1_048_576 * config.maxVideoSize) && !config.whitelist.includes(req.params.id)) { - ws.send('Is this content considered high risk? If so, please email me at admin@preservetube.com.'); - ws.send('This video is too large, and unfortunately, Preservetube does not have unlimited storage.'); - return ws.close() - } else if (!selectedFormats.videoFormat.contentLength) { - ws.send('Youtube isn\'t giving us enough information to be able to tell if we can process this video.') - ws.send('Please try again later.') - return ws.close() + const videoSizeTotal = (selectedFormats.audioFormat.contentLength || 0) + + (selectedFormats.videoFormat.contentLength || 0) + + if (videoSizeTotal > (1_048_576 * config.maxVideoSize) && !config.whitelist.includes(req.params.id)) { + ws.send('Is this content considered high risk? If so, please email me at admin@preservetube.com.'); + ws.send('This video is too large, and unfortunately, Preservetube does not have unlimited storage.'); + return ws.close() + } else if (!selectedFormats.videoFormat.contentLength) { + ws.send('Youtube isn\'t giving us enough information to be able to tell if we can process this video.') + ws.send('Please try again later.') + return ws.close() + } + + ws.send(`VIDEOSIZE-${videoSizeTotal}`) + audioOutputStream = createOutputStream(req.params.id, selectedFormats.audioFormat.mimeType!); + videoOutputStream = createOutputStream(req.params.id, selectedFormats.videoFormat.mimeType!); + + await Promise.all([ + videoStream.pipeTo(createStreamSink(selectedFormats.videoFormat, videoOutputStream.stream, ws, 'video')), + audioStream.pipeTo(createStreamSink(selectedFormats.audioFormat, audioOutputStream.stream, ws, 'audio')) + ]); + } catch (error) { + await lease.refresh() + throw error + } finally { + lease.release() } - - ws.send(`VIDEOSIZE-${videoSizeTotal}`) - audioOutputStream = createOutputStream(req.params.id, selectedFormats.audioFormat.mimeType!); - videoOutputStream = createOutputStream(req.params.id, selectedFormats.videoFormat.mimeType!); - - await Promise.all([ - videoStream.pipeTo(createStreamSink(selectedFormats.videoFormat, videoOutputStream.stream, ws, 'video')), - audioStream.pipeTo(createStreamSink(selectedFormats.audioFormat, audioOutputStream.stream, ws, 'audio')) - ]); } if (audioOutputStream == undefined || videoOutputStream == undefined) { @@ -376,4 +381,4 @@ async function switchIps() { app.listen(8008, () => { console.log('the metadata server is up.') // switchIps() -}) \ No newline at end of file +}) diff --git a/utils/companion.ts b/utils/companion.ts index e6a47f4..eb3ce7f 100644 --- a/utils/companion.ts +++ b/utils/companion.ts @@ -13,7 +13,7 @@ interface StreamResults { export async function getVideoStreams( videoId: string, - adaptiveFormats: any[], + adaptiveFormats: any[], options: { videoQuality: string; audioQuality: string } ): Promise<{ streamResults: StreamResults | false; @@ -24,7 +24,7 @@ export async function getVideoStreams( .sort((a, b) => (parseInt(a.contentLength) || 0) - (parseInt(b.contentLength) || 0))?.[0] const lowestStorageAudio = adaptiveFormats .filter(f => { - if (f.audioTrack?.isAutoDubbed || !f.audioQuality) return false + if (f.audioTrack?.isAutoDubbed || !f.audioQuality) return false if (f.audioTrack && !f.audioTrack.displayName.endsWith('original')) return false return true }) @@ -73,7 +73,7 @@ export async function downloadStream(streamUrl: string, format: any, stream: Wri const config = await Bun.file('config.json').json() let proxy: undefined | string = undefined if (config.proxy) proxy = config.proxy.replace('$$random$$', crypto.randomBytes(4).toString('hex')) - + // get the final url of the stream, since it redirects let location = streamUrl let headResponse: Response | undefined; @@ -109,7 +109,7 @@ export async function downloadStream(streamUrl: string, format: any, stream: Wri throw new Error('google redirected too many times') } - // setup the chunking setup. + // setup the chunking setup. const googleVideoUrl = new URL(location); let size = 0; const totalSize = Number(headResponse.headers.get("Content-Length") || format.content_length || "0") @@ -151,7 +151,7 @@ export async function downloadStream(streamUrl: string, format: any, stream: Wri const chunkSize = 5 * 1_000_000 // 5mb const wholeRequestEndByte = Number(totalSize) - 1; - + for (let startByte = 0; startByte < wholeRequestEndByte; startByte += chunkSize) { let endByte = startByte + chunkSize - 1; if (endByte > wholeRequestEndByte) { diff --git a/utils/sabr-stream-factory.ts b/utils/sabr-stream-factory.ts index 8c22e01..0cbe8d0 100644 --- a/utils/sabr-stream-factory.ts +++ b/utils/sabr-stream-factory.ts @@ -1,5 +1,5 @@ import { createWriteStream, type WriteStream } from 'node:fs'; -import { Constants, Innertube, type IPlayerResponse, UniversalCache, YTNodes } from 'youtubei.js'; +import { Constants, Innertube, type IPlayerResponse, UniversalCache, YTNodes, Platform, type Types } from 'youtubei.js'; import { generateWebPoToken } from './webpo-helper.js'; import type { SabrFormat } from 'googlevideo/shared-types'; @@ -9,6 +9,23 @@ import { buildSabrFormat } from 'googlevideo/utils'; import * as hr from '@tsmx/human-readable' +Platform.shim.eval = async (data: Types.BuildScriptResult, env: Record) => { + const properties = []; + + if(env.n) { + properties.push(`n: exportedVars.nFunction("${env.n}")`) + } + + if (env.sig) { + properties.push(`sig: exportedVars.sigFunction("${env.sig}")`) + } + + const code = `${data.output}\nreturn { ${properties.join(', ')} }`; + + return new Function(code)(); +} + + export interface DownloadOutput { stream: WriteStream; filePath: string; @@ -37,7 +54,7 @@ export async function makePlayerRequest(innertube: Innertube, videoId: string, r vis: 0, splay: false, lactMilliseconds: '-1', - signatureTimestamp: innertube.session.player?.sts + signatureTimestamp: (innertube.session.player as { sts?: number } | undefined)?.sts } }, contentCheckOk: true, @@ -118,12 +135,12 @@ export function createStreamSink(format: SabrFormat, outputStream: WriteStream, */ export async function createSabrStream( videoId: string, - options: SabrPlaybackOptions + options: SabrPlaybackOptions, + innertube: Innertube ): Promise<{ innertube: Innertube; streamResults: StreamResults; }> { - const innertube = await Innertube.create({ cache: new UniversalCache(true) }); const webPoTokenResult = await generateWebPoToken(innertube.session.context.client.visitorData || ''); console.log(`debugging -> ${JSON.stringify(webPoTokenResult)}, ${videoId}`) @@ -132,7 +149,7 @@ export async function createSabrStream( const videoTitle = playerResponse.video_details?.title || 'Unknown Video'; // Now get the streaming information. - const serverAbrStreamingUrl = innertube.session.player?.decipher(playerResponse.streaming_data?.server_abr_streaming_url); + const serverAbrStreamingUrl = await innertube.session.player?.decipher(playerResponse.streaming_data?.server_abr_streaming_url); const videoPlaybackUstreamerConfig = playerResponse.player_config?.media_common_config.media_ustreamer_request_config?.video_playback_ustreamer_config; if (!videoPlaybackUstreamerConfig) throw new Error('ustreamerConfig not found'); @@ -169,7 +186,7 @@ export async function createSabrStream( const videoPlaybackUstreamerConfig = playerResponse.player_config?.media_common_config.media_ustreamer_request_config?.video_playback_ustreamer_config; if (serverAbrStreamingUrl && videoPlaybackUstreamerConfig) { - serverAbrStream.setStreamingURL(serverAbrStreamingUrl); + serverAbrStream.setStreamingURL(await serverAbrStreamingUrl); serverAbrStream.setUstreamerConfig(videoPlaybackUstreamerConfig); } }); @@ -205,4 +222,4 @@ function secondsToTime(seconds: number) { const remainingSeconds = seconds % 60; const formattedSeconds = remainingSeconds < 10 ? '0' + remainingSeconds : remainingSeconds; return `${minutes}:${formattedSeconds}`; -} \ No newline at end of file +}