From fa086e7becb494ec2daf49a5885ad5baa8ab6f8f Mon Sep 17 00:00:00 2001 From: localhost Date: Wed, 1 Oct 2025 21:23:45 +0200 Subject: [PATCH] add option to use non-sabr streams --- config.json | 3 +- index.ts | 100 +++++++++++++++++++-------- utils/companion.ts | 163 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 238 insertions(+), 28 deletions(-) create mode 100644 utils/companion.ts diff --git a/config.json b/config.json index c78774b..4a097fe 100644 --- a/config.json +++ b/config.json @@ -1,4 +1,5 @@ { "whitelist": [], - "player_id": "0004de42" + "player_id": "0004de42", + "useCompanion": true } \ No newline at end of file diff --git a/index.ts b/index.ts index 6d20a8e..f4b699e 100644 --- a/index.ts +++ b/index.ts @@ -8,8 +8,10 @@ import { createOutputStream, createStreamSink, createSabrStream, + type DownloadOutput } from './utils/sabr-stream-factory.js'; import type { SabrPlaybackOptions } from 'googlevideo/sabr-stream'; +import { getVideoStreams, downloadStream } from './utils/companion'; const ffmpeg = require('fluent-ffmpeg') const ffmpegStatic = require('ffmpeg-static') @@ -132,11 +134,12 @@ app.get('/videos/:id', async (req, res) => { // @ts-ignore app.ws('/download/:id', async (ws, req) => { + const config = await Bun.file('config.json').json() const yt = await Innertube.create(); let quality = '480p' const info = await yt.getInfo(req.params.id, { - client: 'IOS' + client: 'TV_SIMPLY' }); if (!info || info.basic_info.duration == undefined) { ws.send('Unable to retrieve video info from YouTube. Please try again later.'); @@ -157,35 +160,78 @@ app.ws('/download/:id', async (ws, req) => { return ws.close() } - const streamOptions: SabrPlaybackOptions = { - videoQuality: quality, - audioQuality: 'AUDIO_QUALITY_LOW', - enabledTrackTypes: EnabledTrackTypes.VIDEO_AND_AUDIO - }; - const { streamResults } = await createSabrStream(req.params.id, streamOptions); - const { videoStream, audioStream, selectedFormats } = streamResults; + let audioOutputStream: DownloadOutput | undefined; + let videoOutputStream: DownloadOutput | undefined; - const config = await Bun.file('config.json').json() - const videoSizeTotal = (selectedFormats.audioFormat.contentLength || 0) - + (selectedFormats.videoFormat.contentLength || 0) + if (config.useCompanion) { + const streamOptions = { + videoQuality: quality, + audioQuality: 'AUDIO_QUALITY_LOW' + }; + const { streamResults, error } = await getVideoStreams(req.params.id, info.streaming_data!.adaptive_formats, streamOptions); + if (streamResults == false) { + ws.send(error) + return ws.close() + } - if (videoSizeTotal > (1_048_576 * 150) && !config.whitelist.includes(req.params.id)) { - ws.send('Is this content considered high risk? If so, please email me at admin@preservetube.com.'); - ws.send('This video is too large, and unfortunately, Preservetube does not have unlimited storage.'); - return ws.close() - } else if (!selectedFormats.videoFormat.contentLength) { - ws.send('Youtube isn\'t giving us enough information to be able to tell if we can process this video.') - ws.send('Please try again later.') - return ws.close() + const { videoStreamUrl, audioStreamUrl, selectedFormats } = streamResults; + + const videoSizeTotal = (selectedFormats.audioFormat.content_length || 0) + + (selectedFormats.videoFormat.content_length || 0) + + if (videoSizeTotal > (1_048_576 * 150) && !config.whitelist.includes(req.params.id)) { + ws.send('Is this content considered high risk? If so, please email me at admin@preservetube.com.'); + ws.send('This video is too large, and unfortunately, Preservetube does not have unlimited storage.'); + return ws.close() + } else if (!selectedFormats.videoFormat.content_length) { + ws.send('Youtube isn\'t giving us enough information to be able to tell if we can process this video.') + ws.send('Please try again later.') + return ws.close() + } + + const audioOutputStream = createOutputStream(req.params.id, selectedFormats.audioFormat.mime_type!); + const videoOutputStream = createOutputStream(req.params.id, selectedFormats.videoFormat.mime_type!); + + await Promise.all([ + downloadStream(videoStreamUrl, selectedFormats.videoFormat, videoOutputStream.stream, ws, 'video'), + downloadStream(audioStreamUrl, selectedFormats.audioFormat, audioOutputStream.stream, ws, 'audio') + ]); + } else { + const streamOptions: SabrPlaybackOptions = { + videoQuality: quality, + audioQuality: 'AUDIO_QUALITY_LOW', + enabledTrackTypes: EnabledTrackTypes.VIDEO_AND_AUDIO + }; + const { streamResults } = await createSabrStream(req.params.id, streamOptions); + const { videoStream, audioStream, selectedFormats } = streamResults; + + const config = await Bun.file('config.json').json() + const videoSizeTotal = (selectedFormats.audioFormat.contentLength || 0) + + (selectedFormats.videoFormat.contentLength || 0) + + if (videoSizeTotal > (1_048_576 * 150) && !config.whitelist.includes(req.params.id)) { + ws.send('Is this content considered high risk? If so, please email me at admin@preservetube.com.'); + ws.send('This video is too large, and unfortunately, Preservetube does not have unlimited storage.'); + return ws.close() + } else if (!selectedFormats.videoFormat.contentLength) { + ws.send('Youtube isn\'t giving us enough information to be able to tell if we can process this video.') + ws.send('Please try again later.') + return ws.close() + } + + const audioOutputStream = createOutputStream(req.params.id, selectedFormats.audioFormat.mimeType!); + const videoOutputStream = createOutputStream(req.params.id, selectedFormats.videoFormat.mimeType!); + + await Promise.all([ + videoStream.pipeTo(createStreamSink(selectedFormats.videoFormat, videoOutputStream.stream, ws, 'video')), + audioStream.pipeTo(createStreamSink(selectedFormats.audioFormat, audioOutputStream.stream, ws, 'audio')) + ]); } - const audioOutputStream = createOutputStream(req.params.id, selectedFormats.audioFormat.mimeType!); - const videoOutputStream = createOutputStream(req.params.id, selectedFormats.videoFormat.mimeType!); - - await Promise.all([ - videoStream.pipeTo(createStreamSink(selectedFormats.videoFormat, videoOutputStream.stream, ws, 'video')), - audioStream.pipeTo(createStreamSink(selectedFormats.audioFormat, audioOutputStream.stream, ws, 'audio')) - ]); + if (audioOutputStream == undefined || videoOutputStream == undefined) { + ws.send('This should not happen. Please report it via admin@preservetube.com.') + return ws.close() + } ws.send('Downloaded video and audio. Merging them together.') @@ -325,5 +371,5 @@ setInterval(switchIps, 30 * 60000) // 30 minutes app.listen(8008, () => { console.log('the metadata server is up.') - switchIps() + // switchIps() }) \ No newline at end of file diff --git a/utils/companion.ts b/utils/companion.ts new file mode 100644 index 0000000..0879d7f --- /dev/null +++ b/utils/companion.ts @@ -0,0 +1,163 @@ +import { type WriteStream } from 'node:fs'; +import * as hr from '@tsmx/human-readable' + +interface StreamResults { + videoStreamUrl: string; + audioStreamUrl: string; + selectedFormats: { + videoFormat: any; + audioFormat: any; + } +} + +export async function getVideoStreams( + videoId: string, + adaptiveFormats: any[], + options: { videoQuality: string; audioQuality: string } +): Promise<{ + streamResults: StreamResults | false; + error?: string +}> { + const lowestStorageVideo = adaptiveFormats + .filter((format) => !!format.quality_label?.toLowerCase().includes(options.videoQuality?.toLowerCase() || '')) + .sort((a, b) => (a.contentLength || 0) - (b.contentLength || 0))?.[0] + const lowestStorageAudio = adaptiveFormats + .filter((format) => !!format.audio_quality?.toLowerCase().includes(options.audioQuality?.toLowerCase() || '')) + .sort((a, b) => (a.contentLength || 0) - (b.contentLength || 0))?.[0] + const lowestOptions = { + videoFormat: lowestStorageVideo?.itag, + audioFormat: lowestStorageAudio?.itag + } + + if (!lowestOptions.videoFormat || !lowestOptions.audioFormat) { + return { streamResults: false, error: 'Couldn\'t find any suitable download formats.' } + } + + const { + videoStreamUrl, + audioStreamUrl + } = { + videoStreamUrl: await getStreamUrl(videoId, lowestOptions.videoFormat), + audioStreamUrl: await getStreamUrl(videoId, lowestOptions.audioFormat) + } + + if (!videoStreamUrl || !audioStreamUrl) return { streamResults: false, error: 'Failed to fetch streaming URLs from Youtube.' } + + return { + streamResults: { + videoStreamUrl, + audioStreamUrl, + selectedFormats: { + videoFormat: lowestStorageVideo, + audioFormat: lowestStorageAudio + } + }, + } +} + +export async function downloadStream(streamUrl: string, format: any, stream: WriteStream, ws: any, type: string) { + // get the final url of the stream, since it redirects + let location = streamUrl + let headResponse: Response | undefined; + const headersToSend: HeadersInit = { + "accept": "*/*", + "accept-encoding": "gzip, deflate, br, zstd", + "accept-language": "en-us,en;q=0.5", + "origin": "https://www.youtube.com", + "referer": "https://www.youtube.com", + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36' + }; + + for (let i = 0; i < 5; i++) { + const googlevideoResponse: Response = await fetch(location, { + method: "HEAD", + headers: headersToSend, + redirect: "manual", + }); + if (googlevideoResponse.status == 403) { + throw new Error(`403 from google - ${await googlevideoResponse.text()}`) + } + if (googlevideoResponse.headers.has("Location")) { + location = googlevideoResponse.headers.get("Location") as string; + continue; + } else { + headResponse = googlevideoResponse; + break; + } + } + + if (headResponse === undefined) { + throw new Error('google redirected too many times') + } + + // setup the chunking setup. + const googleVideoUrl = new URL(location); + let size = 0; + const totalSize = Number(headResponse.headers.get("Content-Length") || format.content_length || "0") + const videoStartTime = Date.now(); + const videoPrecentages: string[] = [] + + const getChunk = async (start: number, end: number) => { + googleVideoUrl.searchParams.set( + "range", + `${start}-${end}`, + ); + const postResponse = await fetch(googleVideoUrl, { + method: "POST", + body: new Uint8Array([0x78, 0]), // protobuf: { 15: 0 } (no idea what it means but this is what YouTube uses), + headers: headersToSend, + }); + if (postResponse.status !== 200) { + throw new Error("Non-200 response from google servers"); + } + + const chunk = Buffer.from(await postResponse.arrayBuffer()) + stream.write(chunk); + + size += chunk.length; + + if (totalSize > 0) { + let elapsedTime = (Date.now() - videoStartTime) / 1000; + let progress = size / totalSize; + let speedInMBps = (size / (1024 * 1024)) / elapsedTime; + let remainingTime = (totalSize - size) / (speedInMBps * 1024 * 1024); + + if (!videoPrecentages.includes((progress * 100).toFixed(0))) { + videoPrecentages.push((progress * 100).toFixed(0)) + ws.send(`[${type}] ${(progress * 100).toFixed(2)}% of ${hr.fromBytes(totalSize, {})} at ${speedInMBps.toFixed(2)} MB/s ETA ${secondsToTime(parseInt(remainingTime.toFixed(0)))}`) + } + } + }; + + const chunkSize = 5 * 1_000_000 // 5mb + const wholeRequestEndByte = Number(totalSize) - 1; + + for (let startByte = 0; startByte < wholeRequestEndByte; startByte += chunkSize) { + let endByte = startByte + chunkSize - 1; + if (endByte > wholeRequestEndByte) { + endByte = wholeRequestEndByte; + } + await getChunk(startByte, endByte) + } + + stream.end() +} + +async function getStreamUrl(videoId: string, itag: number): Promise { + const req = await fetch(`http://127.0.0.1:8282/companion/latest_version?id=${videoId}&itag=${itag}`, { + redirect: 'manual' + }) + + if (req.status == 302) { + return req.headers.get('Location')! + } + + return false +} + +function secondsToTime(seconds: number) { + const minutes = Math.floor(seconds / 60); + const remainingSeconds = seconds % 60; + const formattedSeconds = remainingSeconds < 10 ? '0' + remainingSeconds : remainingSeconds; + return `${minutes}:${formattedSeconds}`; +} \ No newline at end of file