add option to use non-sabr streams

config.json

@@ -1,4 +1,5 @@
 {
   "whitelist": [],
-  "player_id": "0004de42"
+  "player_id": "0004de42",
+  "useCompanion": true
 }

index.ts

@@ -8,8 +8,10 @@ import {
   createOutputStream,
   createStreamSink,
   createSabrStream,
+  type DownloadOutput
 } from './utils/sabr-stream-factory.js';
 import type { SabrPlaybackOptions } from 'googlevideo/sabr-stream';
+import { getVideoStreams, downloadStream } from './utils/companion';
 
 const ffmpeg = require('fluent-ffmpeg')
 const ffmpegStatic = require('ffmpeg-static')
@@ -132,11 +134,12 @@ app.get('/videos/:id', async (req, res) => {
 
 // @ts-ignore
 app.ws('/download/:id', async (ws, req) => {
+  const config = await Bun.file('config.json').json()
   const yt = await Innertube.create();
   let quality = '480p'
 
   const info = await yt.getInfo(req.params.id, {
-    client: 'IOS'
+    client: 'TV_SIMPLY'
   });
   if (!info || info.basic_info.duration == undefined) {
     ws.send('Unable to retrieve video info from YouTube. Please try again later.');
@@ -157,35 +160,78 @@ app.ws('/download/:id', async (ws, req) => {
     return ws.close()
   } 
 
-  const streamOptions: SabrPlaybackOptions = {
-    videoQuality: quality,
-    audioQuality: 'AUDIO_QUALITY_LOW',
-    enabledTrackTypes: EnabledTrackTypes.VIDEO_AND_AUDIO
-  };
-  const { streamResults } = await createSabrStream(req.params.id, streamOptions);
-  const { videoStream, audioStream, selectedFormats } = streamResults;
+  let audioOutputStream: DownloadOutput | undefined;
+  let videoOutputStream: DownloadOutput | undefined;
 
-  const config = await Bun.file('config.json').json()
-  const videoSizeTotal = (selectedFormats.audioFormat.contentLength || 0) 
-    + (selectedFormats.videoFormat.contentLength || 0)
+  if (config.useCompanion) {
+    const streamOptions = {
+      videoQuality: quality,
+      audioQuality: 'AUDIO_QUALITY_LOW'
+    };
+    const { streamResults, error } = await getVideoStreams(req.params.id, info.streaming_data!.adaptive_formats, streamOptions);
+    if (streamResults == false) {
+      ws.send(error)
+      return ws.close()
+    }
 
-  if (videoSizeTotal > (1_048_576 * 150) && !config.whitelist.includes(req.params.id)) {
-    ws.send('Is this content considered high risk? If so, please email me at admin@preservetube.com.');
-    ws.send('This video is too large, and unfortunately, Preservetube does not have unlimited storage.');
-    return ws.close()
-  } else if (!selectedFormats.videoFormat.contentLength) {
-    ws.send('Youtube isn\'t giving us enough information to be able to tell if we can process this video.')
-    ws.send('Please try again later.')
-    return ws.close()
+    const { videoStreamUrl, audioStreamUrl, selectedFormats } = streamResults;
+
+    const videoSizeTotal = (selectedFormats.audioFormat.content_length || 0) 
+      + (selectedFormats.videoFormat.content_length || 0)
+
+    if (videoSizeTotal > (1_048_576 * 150) && !config.whitelist.includes(req.params.id)) {
+      ws.send('Is this content considered high risk? If so, please email me at admin@preservetube.com.');
+      ws.send('This video is too large, and unfortunately, Preservetube does not have unlimited storage.');
+      return ws.close()
+    } else if (!selectedFormats.videoFormat.content_length) {
+      ws.send('Youtube isn\'t giving us enough information to be able to tell if we can process this video.')
+      ws.send('Please try again later.')
+      return ws.close()
+    }
+
+    const audioOutputStream = createOutputStream(req.params.id, selectedFormats.audioFormat.mime_type!);
+    const videoOutputStream = createOutputStream(req.params.id, selectedFormats.videoFormat.mime_type!);
+
+    await Promise.all([
+      downloadStream(videoStreamUrl, selectedFormats.videoFormat, videoOutputStream.stream, ws, 'video'),
+      downloadStream(audioStreamUrl, selectedFormats.audioFormat, audioOutputStream.stream, ws, 'audio')
+    ]);
+  } else {
+    const streamOptions: SabrPlaybackOptions = {
+      videoQuality: quality,
+      audioQuality: 'AUDIO_QUALITY_LOW',
+      enabledTrackTypes: EnabledTrackTypes.VIDEO_AND_AUDIO
+    };
+    const { streamResults } = await createSabrStream(req.params.id, streamOptions);
+    const { videoStream, audioStream, selectedFormats } = streamResults;
+
+    const config = await Bun.file('config.json').json()
+    const videoSizeTotal = (selectedFormats.audioFormat.contentLength || 0) 
+      + (selectedFormats.videoFormat.contentLength || 0)
+
+    if (videoSizeTotal > (1_048_576 * 150) && !config.whitelist.includes(req.params.id)) {
+      ws.send('Is this content considered high risk? If so, please email me at admin@preservetube.com.');
+      ws.send('This video is too large, and unfortunately, Preservetube does not have unlimited storage.');
+      return ws.close()
+    } else if (!selectedFormats.videoFormat.contentLength) {
+      ws.send('Youtube isn\'t giving us enough information to be able to tell if we can process this video.')
+      ws.send('Please try again later.')
+      return ws.close()
+    }
+
+    const audioOutputStream = createOutputStream(req.params.id, selectedFormats.audioFormat.mimeType!);
+    const videoOutputStream = createOutputStream(req.params.id, selectedFormats.videoFormat.mimeType!);
+
+    await Promise.all([
+      videoStream.pipeTo(createStreamSink(selectedFormats.videoFormat, videoOutputStream.stream, ws, 'video')),
+      audioStream.pipeTo(createStreamSink(selectedFormats.audioFormat, audioOutputStream.stream, ws, 'audio'))
+    ]);
   }
 
-  const audioOutputStream = createOutputStream(req.params.id, selectedFormats.audioFormat.mimeType!);
-  const videoOutputStream = createOutputStream(req.params.id, selectedFormats.videoFormat.mimeType!);
-
-  await Promise.all([
-    videoStream.pipeTo(createStreamSink(selectedFormats.videoFormat, videoOutputStream.stream, ws, 'video')),
-    audioStream.pipeTo(createStreamSink(selectedFormats.audioFormat, audioOutputStream.stream, ws, 'audio'))
-  ]);
+  if (audioOutputStream == undefined || videoOutputStream == undefined) {
+    ws.send('This should not happen. Please report it via admin@preservetube.com.')
+    return ws.close()
+  }
 
   ws.send('Downloaded video and audio. Merging them together.')
 
@@ -325,5 +371,5 @@ setInterval(switchIps, 30 * 60000) // 30 minutes
 
 app.listen(8008, () => {
   console.log('the metadata server is up.')
-  switchIps()
+  // switchIps()
 })

utils/companion.ts

@@ -0,0 +1,163 @@
+import { type WriteStream } from 'node:fs';
+import * as hr from '@tsmx/human-readable'
+
+interface StreamResults {
+  videoStreamUrl: string;
+  audioStreamUrl: string;
+  selectedFormats: {
+    videoFormat: any;
+    audioFormat: any;
+  }
+}
+
+export async function getVideoStreams(
+  videoId: string,
+  adaptiveFormats: any[], 
+  options: { videoQuality: string; audioQuality: string }
+): Promise<{
+  streamResults: StreamResults | false;
+  error?: string
+}> {
+  const lowestStorageVideo = adaptiveFormats
+    .filter((format) => !!format.quality_label?.toLowerCase().includes(options.videoQuality?.toLowerCase() || ''))
+    .sort((a, b) => (a.contentLength || 0) - (b.contentLength || 0))?.[0]
+  const lowestStorageAudio = adaptiveFormats
+    .filter((format) => !!format.audio_quality?.toLowerCase().includes(options.audioQuality?.toLowerCase() || ''))
+    .sort((a, b) => (a.contentLength || 0) - (b.contentLength || 0))?.[0]
+  const lowestOptions = {
+    videoFormat: lowestStorageVideo?.itag,
+    audioFormat: lowestStorageAudio?.itag
+  }
+
+  if (!lowestOptions.videoFormat || !lowestOptions.audioFormat) {
+    return { streamResults: false, error: 'Couldn\'t find any suitable download formats.' }
+  }
+
+  const {
+    videoStreamUrl,
+    audioStreamUrl
+  } = {
+    videoStreamUrl: await getStreamUrl(videoId, lowestOptions.videoFormat),
+    audioStreamUrl: await getStreamUrl(videoId, lowestOptions.audioFormat)
+  }
+
+  if (!videoStreamUrl || !audioStreamUrl) return { streamResults: false, error: 'Failed to fetch streaming URLs from Youtube.' }
+
+  return {
+    streamResults: {
+      videoStreamUrl,
+      audioStreamUrl,
+      selectedFormats: {
+        videoFormat: lowestStorageVideo,
+        audioFormat: lowestStorageAudio
+      }
+    },
+  }
+}
+
+export async function downloadStream(streamUrl: string, format: any, stream: WriteStream, ws: any, type: string) {
+  // get the final url of the stream, since it redirects
+  let location = streamUrl
+  let headResponse: Response | undefined;
+  const headersToSend: HeadersInit = {
+    "accept": "*/*",
+    "accept-encoding": "gzip, deflate, br, zstd",
+    "accept-language": "en-us,en;q=0.5",
+    "origin": "https://www.youtube.com",
+    "referer": "https://www.youtube.com",
+    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36'
+  };
+
+  for (let i = 0; i < 5; i++) {
+    const googlevideoResponse: Response = await fetch(location, {
+      method: "HEAD",
+      headers: headersToSend,
+      redirect: "manual",
+    });
+    if (googlevideoResponse.status == 403) {
+      throw new Error(`403 from google - ${await googlevideoResponse.text()}`)
+    }
+    if (googlevideoResponse.headers.has("Location")) {
+      location = googlevideoResponse.headers.get("Location") as string;
+      continue;
+    } else {
+      headResponse = googlevideoResponse;
+      break;
+    }
+  }
+
+  if (headResponse === undefined) {
+    throw new Error('google redirected too many times')
+  }
+
+  // setup the chunking setup. 
+  const googleVideoUrl = new URL(location);
+  let size = 0;
+  const totalSize = Number(headResponse.headers.get("Content-Length") || format.content_length || "0")
+  const videoStartTime = Date.now();
+  const videoPrecentages: string[] = []
+
+  const getChunk = async (start: number, end: number) => {
+    googleVideoUrl.searchParams.set(
+      "range",
+      `${start}-${end}`,
+    );
+    const postResponse = await fetch(googleVideoUrl, {
+      method: "POST",
+      body: new Uint8Array([0x78, 0]), // protobuf: { 15: 0 } (no idea what it means but this is what YouTube uses),
+      headers: headersToSend,
+    });
+    if (postResponse.status !== 200) {
+      throw new Error("Non-200 response from google servers");
+    }
+
+    const chunk = Buffer.from(await postResponse.arrayBuffer())
+    stream.write(chunk);
+
+    size += chunk.length;
+
+    if (totalSize > 0) {
+      let elapsedTime = (Date.now() - videoStartTime) / 1000;
+      let progress = size / totalSize;
+      let speedInMBps = (size / (1024 * 1024)) / elapsedTime;
+      let remainingTime = (totalSize - size) / (speedInMBps * 1024 * 1024);
+
+      if (!videoPrecentages.includes((progress * 100).toFixed(0))) {
+        videoPrecentages.push((progress * 100).toFixed(0))
+        ws.send(`[${type}] ${(progress * 100).toFixed(2)}% of ${hr.fromBytes(totalSize, {})} at ${speedInMBps.toFixed(2)} MB/s ETA ${secondsToTime(parseInt(remainingTime.toFixed(0)))}`)
+      }
+    }
+  };
+
+  const chunkSize = 5 * 1_000_000 // 5mb
+  const wholeRequestEndByte = Number(totalSize) - 1;
+    
+  for (let startByte = 0; startByte < wholeRequestEndByte; startByte += chunkSize) {
+    let endByte = startByte + chunkSize - 1;
+    if (endByte > wholeRequestEndByte) {
+      endByte = wholeRequestEndByte;
+    }
+    await getChunk(startByte, endByte)
+  }
+
+  stream.end()
+}
+
+async function getStreamUrl(videoId: string, itag: number): Promise<string|false> {
+  const req = await fetch(`http://127.0.0.1:8282/companion/latest_version?id=${videoId}&itag=${itag}`, {
+    redirect: 'manual'
+  })
+
+  if (req.status == 302) {
+    return req.headers.get('Location')!
+  }
+
+  return false
+}
+
+function secondsToTime(seconds: number) {
+  const minutes = Math.floor(seconds / 60);
+  const remainingSeconds = seconds % 60;
+  const formattedSeconds = remainingSeconds < 10 ? '0' + remainingSeconds : remainingSeconds;
+  return `${minutes}:${formattedSeconds}`;
+}