From da0c9c23f5c781f3eeaae2964d64ecc37b22a42b Mon Sep 17 00:00:00 2001 From: localhost Date: Thu, 23 Jul 2026 22:22:33 +0200 Subject: [PATCH] avoid channel fetching problem when uploading --- src/router/websocket.ts | 26 ++++++++++++++++++-------- src/utils/common.ts | 6 ++---- src/utils/metadata.ts | 27 ++++++++++++++++++++++++--- 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/src/router/websocket.ts b/src/router/websocket.ts index b71caf6..1394e46 100644 --- a/src/router/websocket.ts +++ b/src/router/websocket.ts @@ -7,7 +7,7 @@ import { validateVideo, validateChannel } from '@/utils/regex' import { checkCaptcha, createDatabaseVideo } from '@/utils/common'; import { downloadVideo } from '@/utils/download'; import { uploadVideo } from '@/utils/upload'; -import { getChannelVideos, getVideo } from '@/utils/metadata'; +import { getChannel, getChannelVideos, getVideo } from '@/utils/metadata'; import { error } from '@/utils/html' import redis from '@/utils/redis'; import { parseSlop } from '@/utils/slop'; @@ -68,7 +68,7 @@ const cleanup = async (ws: any, videoId: string) => { await redis.del(ws.id); }; -const handleUpload = async (ws: any, videoId: string, isChannel: boolean = false) => { +const handleUpload = async (ws: any, videoId: string, metadata: { data: any, channelData: any }, isChannel: boolean = false) => { // the pattern of files that have finished downloading is [videoid].mp4, but some extensions are also possible due to // current youtube changes, so we need to make sure the other extensions are also covered let filePath = fs.readdirSync('./videos/').find(f => f.includes(`${videoId}.`)) @@ -84,7 +84,7 @@ const handleUpload = async (ws: any, videoId: string, isChannel: boolean = false const videoUrl = await uploadVideo(filePath); fs.unlinkSync(filePath); - const uploaded = await createDatabaseVideo(videoId, videoUrl); + const uploaded = await createDatabaseVideo(videoId, videoUrl, metadata); if (uploaded !== 'success') { ws.send(`DATA - Error while uploading - ${JSON.stringify(uploaded)}`); return false; @@ -185,6 +185,10 @@ app.ws('/save', { if (data.error) { return sendError(ws, 'Unable to retrieve video info from YouTube. Please try again later.') } + const channelData = await getChannel(data.videoDetails.channelId) + if (channelData.error) { + return sendError(ws, 'Unable to retrieve video info from YouTube. Please try again later.') + } const isSlop = await parseSlop(videoId, data.videoDetails.title, (data.microformat.playerMicroformatRenderer.description?.simpleText || '').replaceAll('\n', '
'), @@ -217,7 +221,7 @@ app.ws('/save', { } } - const uploadSuccess = await handleUpload(ws, videoId); + const uploadSuccess = await handleUpload(ws, videoId, { data, channelData }); if (!uploadSuccess) await redis.del(saveKey(videoId)); await cleanup(ws, videoId); @@ -273,8 +277,8 @@ app.ws('/savechannel', { } videoIds[ws.id] = `downloading-${channelId}`; - const videos = await getChannelVideos(channelId); - if (!Array.isArray(videos)) { + const [videos, channelData] = await Promise.all([getChannelVideos(channelId), getChannel(channelId)]); + if (!Array.isArray(videos) || channelData.error) { await cleanup(ws, channelId); return sendError(ws, 'Unable to retrieve channel videos from YouTube. Please try again later.'); } @@ -299,6 +303,11 @@ app.ws('/savechannel', { } console.log(`saving (${subjects.map(subject => Bun.hash(subject).toString()).join(',')}) - ${ws.data.path} - ${video.videoId}`) + const data = await getVideo(video.videoId) + if (data.error) { + ws.send(`DATA - Unable to retrieve video info for ${video.videoId}. Skipping.`) + continue + } const isSlop = await parseSlop(video.videoId, video.title, video.description || video.description_snippet?.text || '', channelId) @@ -323,11 +332,12 @@ app.ws('/savechannel', { sendError(ws, limitStatus.isNewVisitorLimited ? NEW_VISITOR_STORAGE_LIMIT_MESSAGE : DEFAULT_STORAGE_LIMIT_MESSAGE, false); break; } - await handleUpload(ws, video.videoId, true); + const uploadSuccess = await handleUpload(ws, video.videoId, { data, channelData }, true); + if (uploadSuccess) ws.send(`DATA - Created video page for ${video.title}`) } await redis.del(saveKey(video.videoId)); - ws.send(`DATA - Created video page for ${video.title}`) + if (downloadResult.fail) continue } await cleanup(ws, channelId); diff --git a/src/utils/common.ts b/src/utils/common.ts index 534bf4e..b4b3c06 100644 --- a/src/utils/common.ts +++ b/src/utils/common.ts @@ -1,4 +1,3 @@ -import { getVideo, getChannel } from "@/utils/metadata"; import { uploadImage } from "@/utils/upload"; import { db } from '@/utils/database' import crypto from 'node:crypto'; @@ -52,9 +51,8 @@ async function checkCaptcha(response: string, remoteIp: string): Promise { return confirm } -async function createDatabaseVideo(id: string, videoUrl: string) { - const data = await getVideo(id) - const channelData = await getChannel(data.videoDetails.channelId) +async function createDatabaseVideo(id: string, videoUrl: string, metadata: { data: any, channelData: any }) { + const { data, channelData } = metadata if (data.error) return data if (channelData.error) return channelData diff --git a/src/utils/metadata.ts b/src/utils/metadata.ts index 253460c..eb6f3b5 100644 --- a/src/utils/metadata.ts +++ b/src/utils/metadata.ts @@ -1,16 +1,37 @@ // metadata either returns innertube or { error: string } import { getMetadataBackend } from '@/utils/health'; +import { sleep } from 'bun'; + +async function getMetadata(path: string, retries = 3, delay = 1000) { + let lastError = { error: 'ErrorUnknown' } + + for (let attempt = 1; attempt <= retries; attempt++) { + try { + const response = await fetch(`${getMetadataBackend()}${path}`) + const json = await response.json() + + if (response.ok && json.error !== 'ErrorUnknown' && json.error !== 'ErrorCantConnectToServiceAPI') return json + if (json.error) lastError = json + } catch { + lastError = { error: 'ErrorUnknown' } + } + + if (attempt < retries) await sleep(delay * attempt) + } + + return lastError +} async function getVideo(id: string) { - return await (await fetch(`${getMetadataBackend()}/video/${id}`)).json() + return await getMetadata(`/video/${id}`) } async function getChannel(id: string) { - return await (await fetch(`${getMetadataBackend()}/channel/${id}`)).json() + return await getMetadata(`/channel/${id}`) } async function getChannelVideos(id: string) { - return await (await fetch(`${getMetadataBackend()}/videos/${id}`)).json() + return await getMetadata(`/videos/${id}`) } export { getVideo, getChannel, getChannelVideos } \ No newline at end of file