add youtubei.js pool

index.ts

@@ -12,6 +12,7 @@ import {
 } from './utils/sabr-stream-factory.js';
 import type { SabrPlaybackOptions } from 'googlevideo/sabr-stream';
 import { getVideoStreams, downloadStream, getInfo } from './utils/companion';
+import { createInnertubePool } from './utils/innertube-pool.js';
 
 const ffmpeg = require('fluent-ffmpeg')
 
@@ -20,7 +21,7 @@ require('express-ws')(app)
 ffmpeg.setFfmpegPath('/usr/local/bin/ffmpeg')
 
 const maxRetries = 5
-const platforms = ['IOS', 'ANDROID', 'YTSTUDIO_ANDROID', 'YTMUSIC_ANDROID']
+const innertubePool = createInnertubePool(parseInt(Bun.env.INNERTUBE_POOL_SIZE || '4'))
 
 app.get('/health', async (req, res) => {
   try {
@@ -69,56 +70,51 @@ app.get('/video/:id', async (req, res) => {
 })
 
 app.get('/channel/:id', async (req, res) => {
-  let error = ''
+  try {
+    const info = await innertubePool.use(async (innertube) => await innertube.getChannel(req.params.id), maxRetries)
 
-  for (let retries = 0; retries < maxRetries; retries++) {
+    if (!info) {
-    try {
+      return res.json({ error: 'ErrorCantConnectToServiceAPI' })
-      const yt = await Innertube.create();
-      const info = await yt.getChannel(req.params.id);
-
-      if (!info) {
-        error = 'ErrorCantConnectToServiceAPI'
-        continue;
-      }
-      return res.json(info)
-    } catch (error) {
-      continue
     }
-  }
 
-  res.json({ error: error || 'ErrorUnknown' })
+    return res.json(info)
+  } catch {
+    return res.json({ error: 'ErrorUnknown' })
+  }
 })
 
 app.get('/videos/:id', async (req, res) => {
   try {
-    let videos: any[] = [];
+    const videos = await innertubePool.use(async (innertube) => {
+      for (let attempt = 0; attempt < 3; attempt++) {
+        const videos: any[] = []
+        const channel = await innertube.getChannel(req.params.id)
 
-    for (let attempt = 0; attempt < 3; attempt++) {
+        let json = await channel.getVideos()
-      videos = [];
+        videos.push(...json.videos)
 
-      const yt = await Innertube.create();
+        while (json.has_continuation && videos.length < 60) {
-      const channel = await yt.getChannel(req.params.id);
+          json = await getNextPage(json)
+          videos.push(...json.videos)
+        }
 
-      let json = await channel.getVideos();
+        if (videos.length) {
-      videos.push(...json.videos);
+          return videos
-
+        }
-      while (json.has_continuation && videos.length < 60) {
-        json = await getNextPage(json);
-        videos.push(...json.videos);
       }
 
-      if (videos.length) break;
+      return []
-    }
+    }, maxRetries)
 
-    return res.json(videos);
+    return res.json(videos)
   } catch (e: any) {
-    console.log(e);
+    console.log(e)
 
     if (e.message.includes('Tab "videos" not found')) {
-      return res.json([]);
+      return res.json([])
     }
 
-    return res.json(false);
+    return res.json(false)
   }
 
   async function getNextPage(json: any) {
@@ -206,30 +202,39 @@ app.ws('/download/:id', async (ws, req) => {
       audioQuality: 'AUDIO_QUALITY_LOW',
       enabledTrackTypes: EnabledTrackTypes.VIDEO_AND_AUDIO
     };
-    const { streamResults } = await createSabrStream(req.params.id, streamOptions);
+    const lease = await innertubePool.acquire()
-    const { videoStream, audioStream, selectedFormats } = streamResults;
 
-    const videoSizeTotal = (selectedFormats.audioFormat.contentLength || 0)
+    try {
-      + (selectedFormats.videoFormat.contentLength || 0)
+      const { streamResults } = await createSabrStream(req.params.id, streamOptions, lease.innertube)
+      const { videoStream, audioStream, selectedFormats } = streamResults
 
-    if (videoSizeTotal > (1_048_576 * config.maxVideoSize) && !config.whitelist.includes(req.params.id)) {
+      const videoSizeTotal = (selectedFormats.audioFormat.contentLength || 0)
-      ws.send('Is this content considered high risk? If so, please email me at admin@preservetube.com.');
+        + (selectedFormats.videoFormat.contentLength || 0)
-      ws.send('This video is too large, and unfortunately, Preservetube does not have unlimited storage.');
+
-      return ws.close()
+      if (videoSizeTotal > (1_048_576 * config.maxVideoSize) && !config.whitelist.includes(req.params.id)) {
-    } else if (!selectedFormats.videoFormat.contentLength) {
+        ws.send('Is this content considered high risk? If so, please email me at admin@preservetube.com.');
-      ws.send('Youtube isn\'t giving us enough information to be able to tell if we can process this video.')
+        ws.send('This video is too large, and unfortunately, Preservetube does not have unlimited storage.');
-      ws.send('Please try again later.')
+        return ws.close()
-      return ws.close()
+      } else if (!selectedFormats.videoFormat.contentLength) {
+        ws.send('Youtube isn\'t giving us enough information to be able to tell if we can process this video.')
+        ws.send('Please try again later.')
+        return ws.close()
+      }
+
+      ws.send(`VIDEOSIZE-${videoSizeTotal}`)
+      audioOutputStream = createOutputStream(req.params.id, selectedFormats.audioFormat.mimeType!);
+      videoOutputStream = createOutputStream(req.params.id, selectedFormats.videoFormat.mimeType!);
+
+      await Promise.all([
+        videoStream.pipeTo(createStreamSink(selectedFormats.videoFormat, videoOutputStream.stream, ws, 'video')),
+        audioStream.pipeTo(createStreamSink(selectedFormats.audioFormat, audioOutputStream.stream, ws, 'audio'))
+      ]);
+    } catch (error) {
+      await lease.refresh()
+      throw error
+    } finally {
+      lease.release()
     }
-
-    ws.send(`VIDEOSIZE-${videoSizeTotal}`)
-    audioOutputStream = createOutputStream(req.params.id, selectedFormats.audioFormat.mimeType!);
-    videoOutputStream = createOutputStream(req.params.id, selectedFormats.videoFormat.mimeType!);
-
-    await Promise.all([
-      videoStream.pipeTo(createStreamSink(selectedFormats.videoFormat, videoOutputStream.stream, ws, 'video')),
-      audioStream.pipeTo(createStreamSink(selectedFormats.audioFormat, audioOutputStream.stream, ws, 'audio'))
-    ]);
   }
 
   if (audioOutputStream == undefined || videoOutputStream == undefined) {
@@ -376,4 +381,4 @@ async function switchIps() {
 app.listen(8008, () => {
   console.log('the metadata server is up.')
   // switchIps()
-})
+})

utils/companion.ts

@@ -13,7 +13,7 @@ interface StreamResults {
 
 export async function getVideoStreams(
   videoId: string,
-  adaptiveFormats: any[], 
+  adaptiveFormats: any[],
   options: { videoQuality: string; audioQuality: string }
 ): Promise<{
   streamResults: StreamResults | false;
@@ -24,7 +24,7 @@ export async function getVideoStreams(
     .sort((a, b) => (parseInt(a.contentLength) || 0) - (parseInt(b.contentLength) || 0))?.[0]
   const lowestStorageAudio = adaptiveFormats
     .filter(f => {
-      if (f.audioTrack?.isAutoDubbed || !f.audioQuality) return false 
+      if (f.audioTrack?.isAutoDubbed || !f.audioQuality) return false
       if (f.audioTrack && !f.audioTrack.displayName.endsWith('original')) return false
       return true
     })
@@ -73,7 +73,7 @@ export async function downloadStream(streamUrl: string, format: any, stream: Wri
   const config = await Bun.file('config.json').json()
   let proxy: undefined | string = undefined
   if (config.proxy) proxy = config.proxy.replace('$$random$$', crypto.randomBytes(4).toString('hex'))
-  
+
   // get the final url of the stream, since it redirects
   let location = streamUrl
   let headResponse: Response | undefined;
@@ -109,7 +109,7 @@ export async function downloadStream(streamUrl: string, format: any, stream: Wri
     throw new Error('google redirected too many times')
   }
 
-  // setup the chunking setup. 
+  // setup the chunking setup.
   const googleVideoUrl = new URL(location);
   let size = 0;
   const totalSize = Number(headResponse.headers.get("Content-Length") || format.content_length || "0")
@@ -151,7 +151,7 @@ export async function downloadStream(streamUrl: string, format: any, stream: Wri
 
   const chunkSize = 5 * 1_000_000 // 5mb
   const wholeRequestEndByte = Number(totalSize) - 1;
-    
+
   for (let startByte = 0; startByte < wholeRequestEndByte; startByte += chunkSize) {
     let endByte = startByte + chunkSize - 1;
     if (endByte > wholeRequestEndByte) {

utils/sabr-stream-factory.ts

@@ -1,5 +1,5 @@
 import { createWriteStream, type WriteStream } from 'node:fs';
-import { Constants, Innertube, type IPlayerResponse, UniversalCache, YTNodes } from 'youtubei.js';
+import { Constants, Innertube, type IPlayerResponse, UniversalCache, YTNodes, Platform, type Types } from 'youtubei.js';
 
 import { generateWebPoToken } from './webpo-helper.js';
 import type { SabrFormat } from 'googlevideo/shared-types';
@@ -9,6 +9,23 @@ import { buildSabrFormat } from 'googlevideo/utils';
 
 import * as hr from '@tsmx/human-readable'
 
+Platform.shim.eval = async (data: Types.BuildScriptResult, env: Record<string, Types.VMPrimative>) => {
+  const properties = [];
+
+  if(env.n) {
+    properties.push(`n: exportedVars.nFunction("${env.n}")`)
+  }
+
+  if (env.sig) {
+    properties.push(`sig: exportedVars.sigFunction("${env.sig}")`)
+  }
+
+  const code = `${data.output}\nreturn { ${properties.join(', ')} }`;
+
+  return new Function(code)();
+}
+
+
 export interface DownloadOutput {
   stream: WriteStream;
   filePath: string;
@@ -37,7 +54,7 @@ export async function makePlayerRequest(innertube: Innertube, videoId: string, r
         vis: 0,
         splay: false,
         lactMilliseconds: '-1',
-        signatureTimestamp: innertube.session.player?.sts
+        signatureTimestamp: (innertube.session.player as { sts?: number } | undefined)?.sts
       }
     },
     contentCheckOk: true,
@@ -118,12 +135,12 @@ export function createStreamSink(format: SabrFormat, outputStream: WriteStream,
  */
 export async function createSabrStream(
   videoId: string,
-  options: SabrPlaybackOptions
+  options: SabrPlaybackOptions,
+  innertube: Innertube
 ): Promise<{
   innertube: Innertube;
   streamResults: StreamResults;
 }> {
-  const innertube = await Innertube.create({ cache: new UniversalCache(true) });
   const webPoTokenResult = await generateWebPoToken(innertube.session.context.client.visitorData || '');
   console.log(`debugging -> ${JSON.stringify(webPoTokenResult)}, ${videoId}`)
 
@@ -132,7 +149,7 @@ export async function createSabrStream(
   const videoTitle = playerResponse.video_details?.title || 'Unknown Video';
 
   // Now get the streaming information.
-  const serverAbrStreamingUrl = innertube.session.player?.decipher(playerResponse.streaming_data?.server_abr_streaming_url);
+  const serverAbrStreamingUrl = await innertube.session.player?.decipher(playerResponse.streaming_data?.server_abr_streaming_url);
   const videoPlaybackUstreamerConfig = playerResponse.player_config?.media_common_config.media_ustreamer_request_config?.video_playback_ustreamer_config;
 
   if (!videoPlaybackUstreamerConfig) throw new Error('ustreamerConfig not found');
@@ -169,7 +186,7 @@ export async function createSabrStream(
     const videoPlaybackUstreamerConfig = playerResponse.player_config?.media_common_config.media_ustreamer_request_config?.video_playback_ustreamer_config;
 
     if (serverAbrStreamingUrl && videoPlaybackUstreamerConfig) {
-      serverAbrStream.setStreamingURL(serverAbrStreamingUrl);
+      serverAbrStream.setStreamingURL(await serverAbrStreamingUrl);
       serverAbrStream.setUstreamerConfig(videoPlaybackUstreamerConfig);
     }
   });
@@ -205,4 +222,4 @@ function secondsToTime(seconds: number) {
   const remainingSeconds = seconds % 60;
   const formattedSeconds = remainingSeconds < 10 ? '0' + remainingSeconds : remainingSeconds;
   return `${minutes}:${formattedSeconds}`;
-}
+}