diff --git a/cold/checkStillUp.ts b/cold/checkStillUp.ts new file mode 100644 index 0000000..d14c3ed --- /dev/null +++ b/cold/checkStillUp.ts @@ -0,0 +1,262 @@ +import { getVideo } from '@/utils/metadata' +import { db } from '@/utils/database' +import * as fs from 'node:fs' + +type FileEntry = { + id: string + size: number + line: string +} + +type VideoStatus = { + id: string + size: number + title: string | null + stillUp: boolean +} + +const INPUT_PATH = 'ratios3.txt' +const OUTPUT_PATH = 'colds3.txt' +const CONCURRENCY = 2 +const checkOnlyChannelId = false + +function parseOutput(text: string): FileEntry[] { + const entries: FileEntry[] = [] + + for (const line of text.split('\n')) { + const trimmed = line.trim() + if (!trimmed) continue + + const ratioEntry = parseRatioEntry(trimmed, line) + if (ratioEntry) { + entries.push(ratioEntry) + continue + } + + const lsEntry = parseLsEntry(trimmed, line) + if (lsEntry) { + entries.push(lsEntry) + } + } + + return entries +} + +function parseLsEntry(trimmed: string, line: string): FileEntry | null { + const match = trimmed.match( + /^[^\s]+\s+\d+\s+\S+\s+\S+\s+(\d+)\s+[A-Za-z]{3}\s+\d{1,2}\s+(?:\d{4}|\d{2}:\d{2})\s+(.+)$/ + ) + if (!match) return null + + const [, sizeText, filename] = match + const size = Number(sizeText) + if (!Number.isFinite(size)) return null + + const idMatch = filename.match(/^([^.]+)\.\w+$/) + const id = idMatch?.[1] + if (!id) return null + + return { id, size, line } +} + +function parseRatioEntry(trimmed: string, line: string): FileEntry | null { + const columns = trimmed.split('\t') + if (columns.length < 7) return null + + const [rankText, , , sizeText, , , filename] = columns + if (!/^\d+$/.test(rankText)) return null + + const size = Number(sizeText) + if (!Number.isFinite(size)) return null + + const idMatch = filename.match(/^([^.]+)\.\w+$/) + const id = idMatch?.[1] + if (!id) return null + + return { id, size, line } +} + +function formatBytes(bytes: number): string { + const units = ['B', 'KB', 'MB', 'GB', 'TB'] + let value = bytes + let unitIndex = 0 + + while (value >= 1024 && unitIndex < units.length - 1) { + value /= 1024 + unitIndex += 1 + } + + return `${value.toFixed(value >= 100 || unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}` +} + +async function mapConcurrent( + items: T[], + concurrency: number, + mapper: (item: T, index: number) => Promise, +): Promise { + const results = new Array(items.length) + let nextIndex = 0 + + async function worker() { + while (nextIndex < items.length) { + const currentIndex = nextIndex + nextIndex += 1 + results[currentIndex] = await mapper(items[currentIndex], currentIndex) + } + } + + await Promise.all( + Array.from({ length: Math.min(concurrency, items.length) }, () => worker()), + ) + + return results +} + +async function getVideoWithRetry(id: string, retries = 3, delay = 1000): Promise { + for (let attempt = 1; attempt <= retries; attempt++) { + try { + const json = await getVideo(id) + if (json && !json.error) { + return json + } + console.warn(`[Retry ${attempt}/${retries}] getVideo(${id}) returned empty or error:`, json) + } catch (e: any) { + console.error(`[Retry ${attempt}/${retries}] getVideo(${id}) threw:`, e) + } + if (attempt < retries) { + await new Promise((resolve) => setTimeout(resolve, delay * attempt)) + } + } + return null +} + +async function getPageViewsWithRetry(id: string, retries = 3, delay = 1000): Promise { + for (let attempt = 1; attempt <= retries; attempt++) { + try { + const response = await fetch(`https://a.gloe.net/api/stats/preservetube.com/top-stats?period=all&date=2026-04-29&filters=%7B%22page%22%3A%22%2Fwatch%2F${id}%22%7D&with_imported=true&comparison=previous_period&compare_from=undefined&compare_to=undefined&match_day_of_week=true`, { + headers: { + 'cookie': 'logged_in=true; _plausible_key=SFMyNTY.g3QAAAAFbQAAAAtfY3NyZl90b2tlbm0AAAAYN0drdE52TGVzejBCSHVBdmlTdWQzTXdTbQAAAA9jdXJyZW50X3VzZXJfaWRhAW0AAAAJbGFzdF9zZWVuYmnxNDttAAAACmxvZ2luX2Rlc3RkAANuaWxtAAAAEnNlc3Npb25fdGltZW91dF9hdGJqA6pE.q1ENNi4aJLNquuSIo_1cca054Tj9lWPmalOAnMNuK8A' + } + }) + if (response.ok) { + const aJson = await response.json() as any + const topStats = aJson?.['top_stats'] + if (Array.isArray(topStats)) { + const pageViewsObj = topStats.find((s: any) => s.name == 'Total pageviews') + if (pageViewsObj && typeof pageViewsObj.value === 'number') { + return pageViewsObj.value + } + } + } + console.warn(`[Retry ${attempt}/${retries}] Failed to fetch page views for ${id}`) + } catch (e: any) { + console.error(`[Retry ${attempt}/${retries}] Fetch page views for ${id} threw:`, e) + } + if (attempt < retries) { + await new Promise((resolve) => setTimeout(resolve, delay * attempt)) + } + } + return 0 +} + +async function main() { + const text = await Bun.file(INPUT_PATH).text() + const files = parseOutput(text) + + if (files.length === 0) { + throw new Error(`No video entries could be parsed from ${INPUT_PATH}`) + } + + await Bun.write(OUTPUT_PATH, '') + + const statuses = (await mapConcurrent(files, CONCURRENCY, async (file) => { + const isAlreadyCold = await db.selectFrom('files') + .where('videoId', '=', file.id) + .selectAll() + .executeTakeFirst() + + if (isAlreadyCold) return { + id: file.id, + size: file.size, + title: 'already cold', + stillUp: false, + } + + const archivedMetadata = await db.selectFrom('videos') + .where('id', '=', file.id) + .selectAll() + .executeTakeFirst() + + if (!archivedMetadata || archivedMetadata.deletion_stage) { + console.log(`${file.id} on disk, but not in database`) + fs.appendFileSync('missing.txt', `${file.line}\n`) + return false + } + + const channelList = ['UCMnULQ6F6kLDAHxofDWIbrw', 'UCWEoAwbfSQtgB-_OUN-g7gg'] + if (archivedMetadata?.channelId && channelList.includes(archivedMetadata.channelId)) { + console.log(file.id, archivedMetadata?.channelId, 'added based on channel id') + fs.appendFileSync(OUTPUT_PATH, `${file.line}\n`) + return { + id: file.id, + size: file.size, + title: archivedMetadata.title, + stillUp: true + } + } + + if (checkOnlyChannelId) return { + id: file.id, + size: file.size, + title: archivedMetadata.title, + stillUp: true + } + + const json = await getVideoWithRetry(file.id) as { videoDetails?: { title?: string } } | null + console.log(json) + const title = json?.videoDetails?.title ?? null + const stillUp = Boolean(title) + + const pageViews = await getPageViewsWithRetry(file.id) + + console.log(file.id, archivedMetadata?.channelId, archivedMetadata?.title, stillUp, pageViews) + + if ((stillUp && pageViews < 5) || (!stillUp && pageViews < 2)) { + fs.appendFileSync(OUTPUT_PATH, `${file.line}\n`) + } + + const status: VideoStatus = { + id: file.id, + size: file.size, + title, + stillUp, + } + + return status + })).filter(s => s != false) + + const stillUpCount = statuses.filter((status) => status.stillUp).length + const gone = statuses + .filter((status) => !status.stillUp) + .sort((a, b) => b.size - a.size) + + console.log(`Parsed ${files.length} files from ${INPUT_PATH}`) + console.log(`Checked ${files.length} videos`) + console.log(`Still up: ${stillUpCount}`) + console.log(`Gone: ${gone.length}`) + console.log(`Wrote ${OUTPUT_PATH}`) + + if (gone.length === 0) { + return + } + + console.log('') + console.log('Gone videos:') + console.log('') + + for (const video of gone) { + console.log(`${formatBytes(video.size)}\t${video.id}`) + } +} + +await main() diff --git a/cold/generateColdRow.ts b/cold/generateColdRow.ts new file mode 100644 index 0000000..90cd5bc --- /dev/null +++ b/cold/generateColdRow.ts @@ -0,0 +1,200 @@ +import { readdir } from 'node:fs/promises' + +type FfprobeStream = { + codec_name?: string + codec_type?: string + duration?: string + width?: number + height?: number + avg_frame_rate?: string + r_frame_rate?: string +} + +type FfprobeFormat = { + duration?: string +} + +type FfprobeOutput = { + streams?: FfprobeStream[] + format?: FfprobeFormat +} + +function printUsage() { + console.error('Usage: bun run generateColdRow.ts --input-folder=/mnt/cold --prefix=cold/ --suffix=.webm') + console.error('Example: bun run generateColdRow.ts --input-folder=./cold --prefix=cold/ --suffix=.webm') +} + +function formatResolution(width?: number, height?: number): string { + if (!width || !height) return 'unknown' + return `${width}x${height}` +} + +function parseDuration(durationText?: string): number { + if (!durationText) return 0 + const duration = Number(durationText) + return Number.isFinite(duration) ? duration : 0 +} + +function parseFps(rate?: string): number { + if (!rate) return 0 + const [numeratorText, denominatorText] = rate.split('/') + const numerator = Number(numeratorText) + const denominator = denominatorText ? Number(denominatorText) : 1 + + if (!Number.isFinite(numerator) || !Number.isFinite(denominator) || denominator === 0) { + return 0 + } + + return Number((numerator / denominator).toFixed(3)) +} + +async function hashFile(path: string): Promise { + const proc = Bun.spawn(['sha256sum', '--', path], { + stdout: 'pipe', + stderr: 'pipe', + }) + + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + + if (exitCode !== 0) { + throw new Error(stderr.trim() || 'sha256sum failed') + } + + const hash = stdout.trim().split(/\s+/)[0] + if (!hash) throw new Error('sha256sum did not return a hash') + return hash +} + +async function getVideoMetadata(path: string): Promise { + const proc = Bun.spawn([ + 'ffprobe', + '-v', + 'error', + '-print_format', + 'json', + '-show_format', + '-show_streams', + path, + ], { + stdout: 'pipe', + stderr: 'pipe', + }) + + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + + if (exitCode !== 0) { + throw new Error(stderr.trim() || 'ffprobe failed') + } + + return JSON.parse(stdout) as FfprobeOutput +} + +function normalizeDirectory(path: string): string { + return path.endsWith('/') ? path : `${path}/` +} + +function escapeRegExp(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +} + +function readFlagValue(arg: string, longName: string, shortName: string): string | undefined { + if (arg.startsWith(`--${longName}=`)) { + return arg.slice(longName.length + 3) + } + + if (arg.startsWith(`-${shortName}=`)) { + return arg.slice(shortName.length + 2) + } +} + +function parseArgs(args: string[]): { inputFolder: string, suffix: string } { + let inputFolder: string | undefined + let prefix: string | undefined + let suffix: string | undefined + + for (const arg of args) { + inputFolder ??= readFlagValue(arg, 'input-folder', 'input-folder') + inputFolder ??= readFlagValue(arg, 'input', 'input') + prefix ??= readFlagValue(arg, 'prefix', 'prefix') + suffix ??= readFlagValue(arg, 'suffix', 'suffix') + } + + if (!prefix || !suffix) { + printUsage() + process.exit(1) + } + + return { + inputFolder: normalizeDirectory(inputFolder ?? prefix), + suffix, + } +} + +async function readInputIds(inputFolder: string, suffix: string): Promise { + const entries = await readdir(inputFolder, { withFileTypes: true }) + const suffixPattern = new RegExp(`${escapeRegExp(suffix)}$`) + + return entries + .filter(entry => entry.isFile() && entry.name.endsWith(suffix)) + .map(entry => entry.name.replace(suffixPattern, '')) + .sort() +} + +async function generateRow(videoId: string, inputFolder: string, suffix: string) { + const filePath = `${inputFolder}${videoId}${suffix}` + const file = Bun.file(filePath) + + if (!(await file.exists())) { + throw new Error(`File not found for ${videoId}: ${filePath}`) + } + + const filename = filePath.split('/').pop() + if (!filename) { + throw new Error(`Invalid file path for ${videoId}: ${filePath}`) + } + + const [hash, metadata] = await Promise.all([ + hashFile(filePath), + getVideoMetadata(filePath), + ]) + + const videoStream = metadata.streams?.find(stream => stream.codec_type === 'video') + const audioStream = metadata.streams?.find(stream => stream.codec_type === 'audio') + + return { + videoId, + filename, + hash, + hash_algorithm: 'sha256', + size_bytes: file.size, + duration_seconds: parseDuration(videoStream?.duration ?? metadata.format?.duration), + video_codec: videoStream?.codec_name ?? 'unknown', + audio_codec: audioStream?.codec_name ?? 'unknown', + resolution: formatResolution(videoStream?.width, videoStream?.height), + fps: parseFps(videoStream?.avg_frame_rate ?? videoStream?.r_frame_rate), + } +} + +async function main() { + const { inputFolder, suffix } = parseArgs(Bun.argv.slice(2)) + const ids = await readInputIds(inputFolder, suffix) + if (ids.length === 0) { + console.error(`No files ending in ${suffix} found in ${inputFolder}`) + printUsage() + process.exit(1) + } + + for (const videoId of ids) { + console.log(await generateRow(videoId, inputFolder, suffix)) + } +} + +await main() diff --git a/cold/insertColdRow.ts b/cold/insertColdRow.ts new file mode 100644 index 0000000..1fefcb9 --- /dev/null +++ b/cold/insertColdRow.ts @@ -0,0 +1,179 @@ +import { db } from '@/utils/database' +import type { NewFile } from '@/types' + +const DEFAULT_INPUT_PATH = './coldRows.txt' + +function chunkArray(array: T[], size: number): T[][] { + const result: T[][] = [] + for (let i = 0; i < array.length; i += size) { + result.push(array.slice(i, i + size)) + } + return result +} + +function getLineAndColumn(text: string, index: number): { line: number; column: number; context: string } { + const lines = text.slice(0, index).split('\n') + const line = lines.length + const column = lines[lines.length - 1].length + 1 + + const allLines = text.split('\n') + const startLineIdx = Math.max(0, line - 3) + const endLineIdx = Math.min(allLines.length - 1, line + 2) + + const contextLines = allLines.slice(startLineIdx, endLineIdx + 1).map((l, i) => { + const currentLineNum = startLineIdx + i + 1 + const prefix = currentLineNum === line ? '-> ' : ' ' + return `${prefix}${String(currentLineNum).padStart(4, ' ')} | ${l}` + }) + + return { + line, + column, + context: contextLines.join('\n'), + } +} + +function extractObjectBlocks(text: string): string[] { + const blocks: string[] = [] + let depth = 0 + let startIndex = -1 + const openBraceIndices: number[] = [] + + for (let index = 0; index < text.length; index++) { + const char = text[index] + + if (char === '{') { + if (depth === 0) { + startIndex = index + } + depth += 1 + openBraceIndices.push(index) + continue + } + + if (char !== '}') continue + + depth -= 1 + openBraceIndices.pop() + + if (depth < 0) { + const { line, column, context } = getLineAndColumn(text, index) + console.error(`Error: Unexpected closing brace at line ${line}, column ${column}:\n${context}`) + throw new Error(`Unexpected closing brace at position ${index}`) + } + + if (depth === 0 && startIndex !== -1) { + blocks.push(text.slice(startIndex, index + 1)) + startIndex = -1 + } + } + + if (depth !== 0) { + console.error(`Error: Input contains an unterminated object literal (depth: ${depth}).`) + console.error(`There are ${openBraceIndices.length} unclosed '{' brace(s).`) + + const maxToPrint = Math.min(openBraceIndices.length, 3) + for (let i = 0; i < maxToPrint; i++) { + const braceIndex = openBraceIndices[i] + const { line, column, context } = getLineAndColumn(text, braceIndex) + console.error(`\nUnclosed '{' #${i + 1} at line ${line}, column ${column}:`) + console.error(context) + } + + if (openBraceIndices.length > maxToPrint) { + console.error(`\n... and ${openBraceIndices.length - maxToPrint} more unclosed brace(s).`) + } + + throw new Error('Input contains an unterminated object literal') + } + + return blocks +} + +function parseColdRow(block: string): NewFile { + const parsed = Function(`"use strict"; return (${block})`)() + + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('Parsed row is not an object') + } + + return parsed as NewFile +} + +async function loadRows(inputPath: string): Promise { + const text = await Bun.file(inputPath).text() + const blocks = extractObjectBlocks(text) + + if (blocks.length === 0) { + throw new Error(`No row objects found in ${inputPath}`) + } + + return blocks.map(parseColdRow) +} + +async function main() { + const args = Bun.argv.slice(2) + const dryRun = args.includes('--dry-run') + const inputPath = args.find(arg => arg !== '--dry-run') ?? DEFAULT_INPUT_PATH + const rows = await loadRows(inputPath) + const videoIds = [...new Set(rows.map(row => row.videoId))] + + const uniqueHashes = [...new Set(rows.map(r => r.hash))] + const existingFilesChunks = [] + for (const chunk of chunkArray(uniqueHashes, 1000)) { + const filesChunk = await db.selectFrom('files') + .where('hash', 'in', chunk) + .selectAll() + .execute() + existingFilesChunks.push(filesChunk) + } + const existingFiles = existingFilesChunks.flat() + + const rowsToInsert: NewFile[] = [] + for (const row of rows) { + const matches = existingFiles.filter(f => f.hash === row.hash) + const isDuplicate = matches.some(match => { + return Object.keys(row).every(key => { + const k = key as keyof NewFile + return row[k] == (match as any)[k] + }) + }) + + if (!isDuplicate) { + rowsToInsert.push(row) + } + } + + if (dryRun) { + console.log(`Parsed ${rows.length} row(s) from ${inputPath}`) + console.log(`Would insert ${rowsToInsert.length} new row(s) and skip ${rows.length - rowsToInsert.length} duplicate(s)`) + console.log(JSON.stringify(rowsToInsert, null, 2)) + console.log(`Would set deletion_stage to cold_storage for ${videoIds.length} video(s)`) + return + } + + if (rowsToInsert.length > 0) { + for (const chunk of chunkArray(rowsToInsert, 100)) { + await db.insertInto('files') + .values(chunk) + .execute() + } + } + + for (const chunk of chunkArray(videoIds, 1000)) { + await db.updateTable('videos') + .where('id', 'in', chunk) + .set({ deletion_stage: 'cold_storage' }) + .execute() + } + + console.log(`Inserted ${rowsToInsert.length} row(s) from ${inputPath}`) + console.log(`Skipped ${rows.length - rowsToInsert.length} duplicate row(s)`) + console.log(`Set deletion_stage to cold_storage for ${videoIds.length} video(s)`) +} + +try { + await main() +} finally { + await db.destroy() +} diff --git a/cold/ratioMinio.ts b/cold/ratioMinio.ts new file mode 100644 index 0000000..a53b000 --- /dev/null +++ b/cold/ratioMinio.ts @@ -0,0 +1,151 @@ +import { readFile } from "node:fs/promises"; +import { db } from '@/utils/database' + +const INPUT_FILE = "s3.txt"; +const AMSTERDAM_TIMEZONE = "Europe/Amsterdam"; +const MIN_SIZE_BYTES = 75 * 1024 ** 2; // 75 MiB + +const coldFiles = await db.selectFrom('files') + .select(['filename']) + .execute() + +const SIZE_MULTIPLIERS: Record = { + KiB: 1024, + MiB: 1024 ** 2, + GiB: 1024 ** 3, +}; + +type ParsedEntry = { + timestamp: string; + filename: string; + sizeText: string; + sizeBytes: number; + ageDays: number; + ratio: number; +}; + +const linePattern = + /^\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) (UTC|CET|CEST)\]\s+([0-9.]+)(KiB|MiB|GiB)\s+\S+\s+(.+)$/; + +function parseTimestamp(dateTime: string, zone: string): Date { + const [datePart, timePart] = dateTime.split(" "); + const [year, month, day] = datePart.split("-").map(Number); + const [hour, minute, second] = timePart.split(":").map(Number); + + if (zone === "UTC") { + return new Date(Date.UTC(year, month - 1, day, hour, minute, second)); + } + + if (zone === "CET" || zone === "CEST") { + const utcOffsetHours = zone === "CEST" ? 2 : 1; + return new Date(Date.UTC(year, month - 1, day, hour - utcOffsetHours, minute, second)); + } + + const utcGuess = new Date(Date.UTC(year, month - 1, day, hour, minute, second)); + const formatter = new Intl.DateTimeFormat("en-CA", { + timeZone: AMSTERDAM_TIMEZONE, + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + hourCycle: "h23", + }); + + const parts = formatter.formatToParts(utcGuess); + const localYear = Number(parts.find((part) => part.type === "year")?.value); + const localMonth = Number(parts.find((part) => part.type === "month")?.value); + const localDay = Number(parts.find((part) => part.type === "day")?.value); + const localHour = Number(parts.find((part) => part.type === "hour")?.value); + const localMinute = Number(parts.find((part) => part.type === "minute")?.value); + const localSecond = Number(parts.find((part) => part.type === "second")?.value); + + const desiredUtcMs = Date.UTC(year, month - 1, day, hour, minute, second); + const displayedUtcMs = Date.UTC( + localYear, + localMonth - 1, + localDay, + localHour, + localMinute, + localSecond, + ); + + return new Date(utcGuess.getTime() - (displayedUtcMs - desiredUtcMs)); +} + +function formatReferenceTime(date: Date): string { + return new Intl.DateTimeFormat("sv-SE", { + timeZone: AMSTERDAM_TIMEZONE, + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + hourCycle: "h23", + timeZoneName: "short", + }).format(date); +} + +async function main(): Promise { + const input = await readFile(INPUT_FILE, "utf8"); + const now = new Date(); + const entries: ParsedEntry[] = []; + + for (const line of input.split(/\r?\n/)) { + const match = line.match(linePattern); + if (!match) { + continue; + } + + const [, timestampWithoutZone, zone, sizeValueText, sizeUnit, filename] = match; + const timestamp = `${timestampWithoutZone} ${zone}`; + const sizeValue = Number(sizeValueText); + const sizeBytes = sizeValue * SIZE_MULTIPLIERS[sizeUnit]; + const createdAt = parseTimestamp(timestampWithoutZone, zone); + const ageDays = Math.max(now.getTime() - createdAt.getTime(), 0) / 86_400_000; + + entries.push({ + timestamp, + filename, + sizeText: `${sizeValueText}${sizeUnit}`, + sizeBytes, + ageDays, + ratio: sizeBytes * ageDays, + }); + } + + entries.sort( + (left, right) => + right.ratio - left.ratio || + right.sizeBytes - left.sizeBytes || + right.ageDays - left.ageDays || + left.filename.localeCompare(right.filename), + ); + + const lines = [ + `Reference time: ${formatReferenceTime(now)}`, + "Ratio formula: size_bytes * age_days", + "Sorted from biggest ratio to lowest ratio.", + "", + "Rank\tRatio\tAge(days)\tSize(bytes)\tSize\tTimestamp\tFilename", + ...entries + .filter(e => e.sizeBytes >= MIN_SIZE_BYTES && !coldFiles.find(c => c.filename == e.filename)) + .map((entry, index) => + [ + String(index + 1), + entry.ratio.toFixed(2), + entry.ageDays.toFixed(2), + String(Math.round(entry.sizeBytes)), + entry.sizeText, + entry.timestamp, + entry.filename, + ].join("\t"), + ), + ]; + + console.log(lines.join("\n")); +} + +await main(); diff --git a/cold/removeDupes.ts b/cold/removeDupes.ts new file mode 100644 index 0000000..c0abbc8 --- /dev/null +++ b/cold/removeDupes.ts @@ -0,0 +1,37 @@ +import { readFile } from 'fs/promises' +import { db } from '@/utils/database' + +async function main() { + const raw = await readFile('bucket.txt', 'utf-8') + + const bucketFiles = new Set( + raw + .split('\n') + .map((l) => l.trim()) + .filter(Boolean) + .map((l) => l.split(/\s+/).pop()!) + ) + + const dbRows = await db + .selectFrom('files') + .select('filename') + .execute() + + const dbFiles = new Set(dbRows.map((r) => r.filename)) + + const missingInDb = [...bucketFiles].filter((f) => !dbFiles.has(f)) + + for (const m of missingInDb) { + const video = await db.selectFrom('videos') + .selectAll() + .where('id', '=', m.split('.')[0]) + .executeTakeFirstOrThrow() + + console.log((new URL(video.source)).hostname, m) + } +} + +main().catch((err) => { + console.error(err) + process.exit(1) +}) \ No newline at end of file diff --git a/cold/removeDuplicateFiles.ts b/cold/removeDuplicateFiles.ts new file mode 100644 index 0000000..8ec35c5 --- /dev/null +++ b/cold/removeDuplicateFiles.ts @@ -0,0 +1,87 @@ +import { db } from '@/utils/database' +import type { File } from '@/types' + +async function main() { + const args = Bun.argv.slice(2) + const dryRun = args.includes('--dry-run') + + console.log('Fetching all files to find duplicates...') + const allFiles = await db.selectFrom('files').selectAll().execute() + + // Group files by hash + const filesByHash = new Map() + for (const file of allFiles) { + const group = filesByHash.get(file.hash) ?? [] + group.push(file) + filesByHash.set(file.hash, group) + } + + const uuidsToDelete: string[] = [] + let duplicateCount = 0 + + for (const [hash, group] of filesByHash.entries()) { + if (group.length > 1) { + // Find duplicates within the group by comparing properties + const uniqueFiles: File[] = [] + + for (const file of group) { + // Check if `file` is a duplicate of any in `uniqueFiles` + const isDuplicate = uniqueFiles.some(uniqueFile => { + return ( + file.videoId === uniqueFile.videoId && + file.filename === uniqueFile.filename && + file.hash_algorithm === uniqueFile.hash_algorithm && + file.size_bytes == uniqueFile.size_bytes && + file.duration_seconds == uniqueFile.duration_seconds && + file.video_codec === uniqueFile.video_codec && + file.audio_codec === uniqueFile.audio_codec && + file.resolution === uniqueFile.resolution && + file.fps == uniqueFile.fps + ) + }) + + if (isDuplicate) { + uuidsToDelete.push(file.uuid as string) + duplicateCount++ + } else { + uniqueFiles.push(file) + } + } + } + } + + if (uuidsToDelete.length === 0) { + console.log('No duplicate files found.') + return + } + + console.log(`Found ${duplicateCount} duplicate file(s) across ${filesByHash.size} unique hash(es).`) + + if (dryRun) { + console.log(`\n[DRY RUN] Would delete the following ${uuidsToDelete.length} UUID(s):`) + console.log(uuidsToDelete.join('\n')) + console.log('\nRun without --dry-run to execute the deletion.') + return + } + + console.log(`Deleting ${uuidsToDelete.length} duplicate file(s)...`) + + // Chunk the deletions in case there are many + const chunkSize = 1000 + for (let i = 0; i < uuidsToDelete.length; i += chunkSize) { + const chunk = uuidsToDelete.slice(i, i + chunkSize) + await db.deleteFrom('files') + .where('uuid', 'in', chunk) + .execute() + } + + console.log('Successfully deleted duplicate files.') +} + +try { + await main() +} catch (error) { + console.error(error) +} finally { + await db.destroy() +} diff --git a/cold/sortDeletion.ts b/cold/sortDeletion.ts new file mode 100644 index 0000000..46ac1dc --- /dev/null +++ b/cold/sortDeletion.ts @@ -0,0 +1,69 @@ +import { db } from '@/utils/database' +import * as fs from 'node:fs' + +const target: string = 's0' +let totalSize = 0 + +const bucket = (await Bun.file('bucket.txt').text()).split('\n') +const alreadyRemoved = (await Bun.file('headRemoved.txt').text()).split('\n') +const rmFiles = [] + +const videos = await db.selectFrom('videos') + .where('source', 'like', `%${target}.archive.party%`) + .where('deletion_stage', '=', 'cold_storage') + .select(['id', 'source']) + .execute() + +const files = await db.selectFrom('files') + .where('videoId', 'in', videos.map(v => v.id)) + .orderBy('files.size_bytes desc') + .selectAll() + .execute() + +for (const f of files) { + if (alreadyRemoved.includes(f.filename)) continue + + let isFlagged = false + const bucketSize = bucket.find(b => b.includes(f.filename))?.split(' ').filter(Boolean)[2] + if (!bucketSize) { + console.log('file not in bucket', f.filename) + continue + } + + const video = videos.find(v => v.id == f.videoId) + const headReq = await fetch(video!.source + .replace('https://s0.archive.party', 'http://193.24.209.204:1337') + .replace('https://minio.archive.party', 'http://5.83.147.250:9000') + .replace('https://s2.archive.party', 'http://185.207.125.54:9000') + .replace('https://s3.archive.party', 'http://193.24.209.233:8080'), + { + method: 'HEAD' + }) + + if (headReq.status == 404) { + fs.appendFileSync('headRemoved.txt', f.filename + '\n') + continue + } + if (headReq.headers.get('Content-Length') != bucketSize) { + console.log('head size not same for', f.filename) + console.log([headReq.headers.get('Content-Length'), bucketSize]) + isFlagged = true + } + + if (f.size_bytes.toString() != bucketSize) { + console.log('size not same for', f.filename) + console.log(f.size_bytes, bucketSize) + isFlagged = true + } + + if (isFlagged) continue + console.log(f.size_bytes / 1e9, f.filename + '\n') + totalSize += f.size_bytes / 1e9 + + if (f.size_bytes / 1e9 > 0) rmFiles.push(f.filename) + if (f.size_bytes / 1e9 < 0) break; +} + +console.log(totalSize) +if (target == 'minio' || target == 's2' || target == 's3') console.log(`./mc rm ${rmFiles.map(r => `local/preservetube/${r}`).join(' ')}`) +if (target == 's0') console.log(`rm -- ${rmFiles.join(' ')}`) \ No newline at end of file diff --git a/cold/sumColdSizes.sh b/cold/sumColdSizes.sh new file mode 100755 index 0000000..8afd228 --- /dev/null +++ b/cold/sumColdSizes.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +input_file="${1:-cold.txt}" + +cat "$input_file" | awk ' +function multiplier(unit) { + if (unit == "KiB") return 1024 + if (unit == "MiB") return 1024 * 1024 + if (unit == "GiB") return 1024 * 1024 * 1024 + return 0 +} + +function human(bytes) { + if (bytes >= 1024 * 1024 * 1024) return sprintf("%.2fGiB", bytes / (1024 * 1024 * 1024)) + if (bytes >= 1024 * 1024) return sprintf("%.2fMiB", bytes / (1024 * 1024)) + if (bytes >= 1024) return sprintf("%.2fKiB", bytes / 1024) + return sprintf("%dB", bytes) +} + +{ + size = $5 + if (size ~ /^[0-9.]+(KiB|MiB|GiB)$/) { + value = size + unit = size + sub(/(KiB|MiB|GiB)$/, "", value) + sub(/^[0-9.]+/, "", unit) + total += value * multiplier(unit) + parsed++ + } +} + +END { + printf "Parsed lines: %d\n", parsed + printf "Total bytes: %.0f\n", total + printf "Total size: %s\n", human(total) +} +' diff --git a/generateColdRow.ts b/generateColdRow.ts new file mode 100644 index 0000000..90cd5bc --- /dev/null +++ b/generateColdRow.ts @@ -0,0 +1,200 @@ +import { readdir } from 'node:fs/promises' + +type FfprobeStream = { + codec_name?: string + codec_type?: string + duration?: string + width?: number + height?: number + avg_frame_rate?: string + r_frame_rate?: string +} + +type FfprobeFormat = { + duration?: string +} + +type FfprobeOutput = { + streams?: FfprobeStream[] + format?: FfprobeFormat +} + +function printUsage() { + console.error('Usage: bun run generateColdRow.ts --input-folder=/mnt/cold --prefix=cold/ --suffix=.webm') + console.error('Example: bun run generateColdRow.ts --input-folder=./cold --prefix=cold/ --suffix=.webm') +} + +function formatResolution(width?: number, height?: number): string { + if (!width || !height) return 'unknown' + return `${width}x${height}` +} + +function parseDuration(durationText?: string): number { + if (!durationText) return 0 + const duration = Number(durationText) + return Number.isFinite(duration) ? duration : 0 +} + +function parseFps(rate?: string): number { + if (!rate) return 0 + const [numeratorText, denominatorText] = rate.split('/') + const numerator = Number(numeratorText) + const denominator = denominatorText ? Number(denominatorText) : 1 + + if (!Number.isFinite(numerator) || !Number.isFinite(denominator) || denominator === 0) { + return 0 + } + + return Number((numerator / denominator).toFixed(3)) +} + +async function hashFile(path: string): Promise { + const proc = Bun.spawn(['sha256sum', '--', path], { + stdout: 'pipe', + stderr: 'pipe', + }) + + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + + if (exitCode !== 0) { + throw new Error(stderr.trim() || 'sha256sum failed') + } + + const hash = stdout.trim().split(/\s+/)[0] + if (!hash) throw new Error('sha256sum did not return a hash') + return hash +} + +async function getVideoMetadata(path: string): Promise { + const proc = Bun.spawn([ + 'ffprobe', + '-v', + 'error', + '-print_format', + 'json', + '-show_format', + '-show_streams', + path, + ], { + stdout: 'pipe', + stderr: 'pipe', + }) + + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + + if (exitCode !== 0) { + throw new Error(stderr.trim() || 'ffprobe failed') + } + + return JSON.parse(stdout) as FfprobeOutput +} + +function normalizeDirectory(path: string): string { + return path.endsWith('/') ? path : `${path}/` +} + +function escapeRegExp(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +} + +function readFlagValue(arg: string, longName: string, shortName: string): string | undefined { + if (arg.startsWith(`--${longName}=`)) { + return arg.slice(longName.length + 3) + } + + if (arg.startsWith(`-${shortName}=`)) { + return arg.slice(shortName.length + 2) + } +} + +function parseArgs(args: string[]): { inputFolder: string, suffix: string } { + let inputFolder: string | undefined + let prefix: string | undefined + let suffix: string | undefined + + for (const arg of args) { + inputFolder ??= readFlagValue(arg, 'input-folder', 'input-folder') + inputFolder ??= readFlagValue(arg, 'input', 'input') + prefix ??= readFlagValue(arg, 'prefix', 'prefix') + suffix ??= readFlagValue(arg, 'suffix', 'suffix') + } + + if (!prefix || !suffix) { + printUsage() + process.exit(1) + } + + return { + inputFolder: normalizeDirectory(inputFolder ?? prefix), + suffix, + } +} + +async function readInputIds(inputFolder: string, suffix: string): Promise { + const entries = await readdir(inputFolder, { withFileTypes: true }) + const suffixPattern = new RegExp(`${escapeRegExp(suffix)}$`) + + return entries + .filter(entry => entry.isFile() && entry.name.endsWith(suffix)) + .map(entry => entry.name.replace(suffixPattern, '')) + .sort() +} + +async function generateRow(videoId: string, inputFolder: string, suffix: string) { + const filePath = `${inputFolder}${videoId}${suffix}` + const file = Bun.file(filePath) + + if (!(await file.exists())) { + throw new Error(`File not found for ${videoId}: ${filePath}`) + } + + const filename = filePath.split('/').pop() + if (!filename) { + throw new Error(`Invalid file path for ${videoId}: ${filePath}`) + } + + const [hash, metadata] = await Promise.all([ + hashFile(filePath), + getVideoMetadata(filePath), + ]) + + const videoStream = metadata.streams?.find(stream => stream.codec_type === 'video') + const audioStream = metadata.streams?.find(stream => stream.codec_type === 'audio') + + return { + videoId, + filename, + hash, + hash_algorithm: 'sha256', + size_bytes: file.size, + duration_seconds: parseDuration(videoStream?.duration ?? metadata.format?.duration), + video_codec: videoStream?.codec_name ?? 'unknown', + audio_codec: audioStream?.codec_name ?? 'unknown', + resolution: formatResolution(videoStream?.width, videoStream?.height), + fps: parseFps(videoStream?.avg_frame_rate ?? videoStream?.r_frame_rate), + } +} + +async function main() { + const { inputFolder, suffix } = parseArgs(Bun.argv.slice(2)) + const ids = await readInputIds(inputFolder, suffix) + if (ids.length === 0) { + console.error(`No files ending in ${suffix} found in ${inputFolder}`) + printUsage() + process.exit(1) + } + + for (const videoId of ids) { + console.log(await generateRow(videoId, inputFolder, suffix)) + } +} + +await main() diff --git a/insertColdRow.ts b/insertColdRow.ts new file mode 100644 index 0000000..1fefcb9 --- /dev/null +++ b/insertColdRow.ts @@ -0,0 +1,179 @@ +import { db } from '@/utils/database' +import type { NewFile } from '@/types' + +const DEFAULT_INPUT_PATH = './coldRows.txt' + +function chunkArray(array: T[], size: number): T[][] { + const result: T[][] = [] + for (let i = 0; i < array.length; i += size) { + result.push(array.slice(i, i + size)) + } + return result +} + +function getLineAndColumn(text: string, index: number): { line: number; column: number; context: string } { + const lines = text.slice(0, index).split('\n') + const line = lines.length + const column = lines[lines.length - 1].length + 1 + + const allLines = text.split('\n') + const startLineIdx = Math.max(0, line - 3) + const endLineIdx = Math.min(allLines.length - 1, line + 2) + + const contextLines = allLines.slice(startLineIdx, endLineIdx + 1).map((l, i) => { + const currentLineNum = startLineIdx + i + 1 + const prefix = currentLineNum === line ? '-> ' : ' ' + return `${prefix}${String(currentLineNum).padStart(4, ' ')} | ${l}` + }) + + return { + line, + column, + context: contextLines.join('\n'), + } +} + +function extractObjectBlocks(text: string): string[] { + const blocks: string[] = [] + let depth = 0 + let startIndex = -1 + const openBraceIndices: number[] = [] + + for (let index = 0; index < text.length; index++) { + const char = text[index] + + if (char === '{') { + if (depth === 0) { + startIndex = index + } + depth += 1 + openBraceIndices.push(index) + continue + } + + if (char !== '}') continue + + depth -= 1 + openBraceIndices.pop() + + if (depth < 0) { + const { line, column, context } = getLineAndColumn(text, index) + console.error(`Error: Unexpected closing brace at line ${line}, column ${column}:\n${context}`) + throw new Error(`Unexpected closing brace at position ${index}`) + } + + if (depth === 0 && startIndex !== -1) { + blocks.push(text.slice(startIndex, index + 1)) + startIndex = -1 + } + } + + if (depth !== 0) { + console.error(`Error: Input contains an unterminated object literal (depth: ${depth}).`) + console.error(`There are ${openBraceIndices.length} unclosed '{' brace(s).`) + + const maxToPrint = Math.min(openBraceIndices.length, 3) + for (let i = 0; i < maxToPrint; i++) { + const braceIndex = openBraceIndices[i] + const { line, column, context } = getLineAndColumn(text, braceIndex) + console.error(`\nUnclosed '{' #${i + 1} at line ${line}, column ${column}:`) + console.error(context) + } + + if (openBraceIndices.length > maxToPrint) { + console.error(`\n... and ${openBraceIndices.length - maxToPrint} more unclosed brace(s).`) + } + + throw new Error('Input contains an unterminated object literal') + } + + return blocks +} + +function parseColdRow(block: string): NewFile { + const parsed = Function(`"use strict"; return (${block})`)() + + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('Parsed row is not an object') + } + + return parsed as NewFile +} + +async function loadRows(inputPath: string): Promise { + const text = await Bun.file(inputPath).text() + const blocks = extractObjectBlocks(text) + + if (blocks.length === 0) { + throw new Error(`No row objects found in ${inputPath}`) + } + + return blocks.map(parseColdRow) +} + +async function main() { + const args = Bun.argv.slice(2) + const dryRun = args.includes('--dry-run') + const inputPath = args.find(arg => arg !== '--dry-run') ?? DEFAULT_INPUT_PATH + const rows = await loadRows(inputPath) + const videoIds = [...new Set(rows.map(row => row.videoId))] + + const uniqueHashes = [...new Set(rows.map(r => r.hash))] + const existingFilesChunks = [] + for (const chunk of chunkArray(uniqueHashes, 1000)) { + const filesChunk = await db.selectFrom('files') + .where('hash', 'in', chunk) + .selectAll() + .execute() + existingFilesChunks.push(filesChunk) + } + const existingFiles = existingFilesChunks.flat() + + const rowsToInsert: NewFile[] = [] + for (const row of rows) { + const matches = existingFiles.filter(f => f.hash === row.hash) + const isDuplicate = matches.some(match => { + return Object.keys(row).every(key => { + const k = key as keyof NewFile + return row[k] == (match as any)[k] + }) + }) + + if (!isDuplicate) { + rowsToInsert.push(row) + } + } + + if (dryRun) { + console.log(`Parsed ${rows.length} row(s) from ${inputPath}`) + console.log(`Would insert ${rowsToInsert.length} new row(s) and skip ${rows.length - rowsToInsert.length} duplicate(s)`) + console.log(JSON.stringify(rowsToInsert, null, 2)) + console.log(`Would set deletion_stage to cold_storage for ${videoIds.length} video(s)`) + return + } + + if (rowsToInsert.length > 0) { + for (const chunk of chunkArray(rowsToInsert, 100)) { + await db.insertInto('files') + .values(chunk) + .execute() + } + } + + for (const chunk of chunkArray(videoIds, 1000)) { + await db.updateTable('videos') + .where('id', 'in', chunk) + .set({ deletion_stage: 'cold_storage' }) + .execute() + } + + console.log(`Inserted ${rowsToInsert.length} row(s) from ${inputPath}`) + console.log(`Skipped ${rows.length - rowsToInsert.length} duplicate row(s)`) + console.log(`Set deletion_stage to cold_storage for ${videoIds.length} video(s)`) +} + +try { + await main() +} finally { + await db.destroy() +}