Add manual inbox ingestion

src/router/admin.ts

@@ -12,6 +12,7 @@ import {
 import { startRestore, RestoreError } from '@/utils/restore'
 import { approveRequest, rejectRequest, dismissRequest } from '@/utils/archiveRequests'
 import { approveColdRequest, rejectColdRequest, dismissColdRequest } from '@/utils/coldRequests'
+import { ingestInboxMail, listInboxMail } from '@/utils/inbox'
 
 const app = new Elysia({ prefix: '/admin' })
 
@@ -114,6 +115,23 @@ app.get('/', async ({ set }) => {
   }))
 })
 
+app.get('/inbox', async ({ set }) => {
+  set.headers['Content-Type'] = 'text/html; charset=utf-8'
+  return await m(eta.render('./admin/inbox', {
+    title: 'Inbox | PreserveTube Admin',
+    mails: await listInboxMail()
+  }))
+})
+
+app.post('/inbox/ingest', async ({ body, redirect }) => {
+  await ingestInboxMail(body.uid)
+  return redirect('/admin')
+}, {
+  body: t.Object({
+    uid: t.Numeric()
+  })
+})
+
 app.post('/restore', async ({ body, redirect, error }) => {
   const { videoId, requesterEmail } = body
 

src/templates/admin/dashboard.eta

@@ -8,7 +8,10 @@
     </form>
   </div>
 
-  <h3>Archive Requests</h3>
+  <div class="section-head">
+    <h3>Archive Requests</h3>
+    <form method="GET" action="/admin/inbox"><button type="submit">manually ingest</button></form>
+  </div>
   <p class="stage-desc">Requests found in the admin inbox. Approve to whitelist + archive and email the requester the links. Reject asks the requester for more context (optionally with your note) instead of refusing.</p>
 
   <% it.archiveRequests.forEach(function(r){ %>
@@ -247,6 +250,27 @@
     margin-top: 12px;
   }
 
+  .section-head {
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    gap: 12px;
+    margin-top: 1em;
+  }
+
+  .section-head h3 {
+    margin: 0;
+  }
+
+  .section-head button {
+    padding: 8px 16px;
+    border: 1px solid #1b1c1f;
+    background-color: #1b1c1f;
+    color: white;
+    cursor: pointer;
+    font-family: inherit;
+  }
+
   .admin-link-button {
     background: none;
     border: none;

src/templates/admin/inbox.eta

@@ -0,0 +1,52 @@
+<% layout('../layout') %>
+
+<div class="page">
+  <div class="admin-nav">
+    <h2>Admin Inbox</h2>
+    <a class="a" href="/admin">Back to requests</a>
+  </div>
+
+  <p class="stage-desc">Select one of the latest 20 emails in INBOX. Manually ingesting bypasses AI classification, but still requires a valid YouTube video ID.</p>
+
+  <% if (it.mails.length) { %>
+    <form method="POST" action="/admin/inbox/ingest">
+      <table class="admin-table">
+        <thead><tr><th></th><th>Sender</th><th>Subject</th></tr></thead>
+        <tbody>
+          <% it.mails.forEach(function(mail, index){ %>
+            <tr>
+              <td><input type="radio" name="uid" value="<%= mail.uid %>" <%= index === 0 ? 'checked' : '' %> required /></td>
+              <td><%= mail.fromName ? mail.fromName + ' ' : '' %>&lt;<%= mail.fromEmail %>&gt;</td>
+              <td><%= mail.subject %></td>
+            </tr>
+          <% }) %>
+        </tbody>
+      </table>
+      <div class="form-actions"><button type="submit">manually ingest selected email</button></div>
+    </form>
+  <% } %>
+  <% if (it.mails.length === 0) { %><p class="stage-desc">The inbox is empty or the inbox integration is disabled.</p><% } %>
+</div>
+
+<style>
+  .page { margin: 0 auto; padding: 5px 10px 10px; width: 65%; }
+  .admin-nav { display: flex; align-items: center; justify-content: space-between; margin-top: 12px; }
+  .stage-desc { color: #aaa; font-size: 0.9rem; }
+  .admin-table { border-collapse: collapse; width: 100%; }
+  .admin-table th, .admin-table td { border-bottom: 1px solid #eee; padding: 8px; text-align: left; font-size: 0.9rem; }
+  .admin-table tbody tr:has(input:checked) { background-color: #f7f7f7; }
+  .form-actions { margin-top: 12px; }
+  .form-actions button {
+    padding: 8px 16px;
+    border: 1px solid #1b1c1f;
+    background-color: #1b1c1f;
+    color: white;
+    cursor: pointer;
+    font-family: inherit;
+  }
+  .a { text-decoration-line: underline; text-decoration-style: dotted; }
+  @media (max-width: 900px) {
+    .page { width: auto; }
+    .admin-table { display: block; overflow-x: auto; }
+  }
+</style>

src/utils/archiveRequests.ts

@@ -41,6 +41,7 @@ async function createRequest(input: {
   bareIds?: string[]
   // one-time catch-up of old mail: re-classify ignored mail, only cold storage requests count
   backfill?: boolean
+  forceArchive?: boolean
 }): Promise<'created' | 'ignored' | 'duplicate'> {
   const [existingArchive, existingCold] = await Promise.all([
     db.selectFrom('archive_requests').select('uuid').where('message_id', '=', input.messageId).executeTakeFirst(),
@@ -50,9 +51,11 @@ async function createRequest(input: {
 
   // classified as "not a request" before: skip without paying for another llm call
   const ignoredKey = `inbox:ignored:${input.messageId}`
-  if (!input.backfill && await redis.get(ignoredKey)) return 'ignored'
+  if (!input.backfill && !input.forceArchive && await redis.get(ignoredKey)) return 'ignored'
 
-  const verdict = await classifyEmail(input.subject, input.body)
+  const verdict = input.forceArchive
+    ? { category: 'archive' as const, summary: 'Manually ingested from the admin inbox.' }
+    : await classifyEmail(input.subject, input.body)
   // the catch-up scan can't tell what an unclassified mail is: fail so it is retried, not skipped
   if (input.backfill && verdict.failed) throw new Error('AI classification failed')
   if (verdict.category === 'cold_storage') {

src/utils/inbox.ts

@@ -38,6 +38,13 @@ function referencedIds(mail: ParsedMail): string[] {
 // skip: leave it in the inbox for a human
 type MailResult = 'ingested' | 'seen' | 'skip'
 
+interface InboxMail {
+  uid: number
+  fromEmail: string
+  fromName: string | null
+  subject: string
+}
+
 async function processMail(mail: ParsedMail, fallbackId: string, backfill: boolean): Promise<MailResult> {
   const from = mail.from?.value[0]
   if (!from?.address) return 'seen'
@@ -183,6 +190,102 @@ async function pollInbox(backfill = false) {
   }
 }
 
+async function listInboxMail(): Promise<InboxMail[]> {
+  if (!process.env.IMAP_HOST || !process.env.IMAP_USER || !process.env.IMAP_PASS) return []
+
+  const port = Number(process.env.IMAP_PORT || 993)
+  const client = new ImapFlow({
+    host: process.env.IMAP_HOST,
+    port,
+    secure: port === 993,
+    auth: { user: process.env.IMAP_USER, pass: process.env.IMAP_PASS },
+    logger: false
+  })
+  client.on('error', (error: Error) => console.log(`[inbox] connection error: ${error.message}`))
+
+  try {
+    await client.connect()
+    const lock = await client.getMailboxLock('INBOX')
+
+    try {
+      const uids = await client.search({ all: true }, { uid: true }) || []
+      const mails: InboxMail[] = []
+
+      for (const uid of uids.slice(-20).reverse()) {
+        const msg = await client.fetchOne(String(uid), { source: true }, { uid: true })
+        if (!msg || !msg.source) continue
+
+        const mail = await simpleParser(msg.source)
+        mails.push({
+          uid,
+          fromEmail: mail.from?.value[0]?.address || '(unknown sender)',
+          fromName: mail.from?.value[0]?.name || null,
+          subject: mail.subject || '(no subject)'
+        })
+      }
+
+      return mails
+    } finally {
+      lock.release()
+    }
+  } finally {
+    await client.logout().catch(() => {})
+  }
+}
+
+async function ingestInboxMail(uid: number): Promise<'created' | 'duplicate' | 'ignored' | 'missing'> {
+  if (!process.env.IMAP_HOST || !process.env.IMAP_USER || !process.env.IMAP_PASS) return 'missing'
+
+  const port = Number(process.env.IMAP_PORT || 993)
+  const client = new ImapFlow({
+    host: process.env.IMAP_HOST,
+    port,
+    secure: port === 993,
+    auth: { user: process.env.IMAP_USER, pass: process.env.IMAP_PASS },
+    logger: false
+  })
+  client.on('error', (error: Error) => console.log(`[inbox] connection error: ${error.message}`))
+
+  try {
+    await client.connect()
+    await client.mailboxCreate(INGESTED_FOLDER).catch(() => {})
+    const lock = await client.getMailboxLock('INBOX')
+
+    try {
+      const msg = await client.fetchOne(String(uid), { source: true }, { uid: true })
+      if (!msg || !msg.source) return 'missing'
+
+      const mail = await simpleParser(msg.source)
+      const from = mail.from?.value[0]
+      if (!from?.address) return 'ignored'
+
+      const body = bodyOf(mail)
+      const raw = `${mail.text || ''}\n${typeof mail.html === 'string' ? mail.html : ''}`
+      const videoIds = extractUrlIds(raw)
+      const result = await createRequest({
+        messageId: mail.messageId || `<uid-${uid}@${process.env.IMAP_HOST}>`,
+        fromEmail: from.address,
+        fromName: from.name || null,
+        subject: mail.subject || '(no subject)',
+        body,
+        videoIds,
+        bareIds: extractBareIds(mail.text || body, new Set(videoIds)),
+        forceArchive: true
+      })
+
+      if (result !== 'ignored') {
+        await client.messageFlagsAdd(String(uid), ['\\Seen'], { uid: true })
+        await client.messageMove(String(uid), INGESTED_FOLDER, { uid: true })
+      }
+      return result
+    } finally {
+      lock.release()
+    }
+  } finally {
+    await client.logout().catch(() => {})
+  }
+}
+
 async function startInboxPoller() {
   if (!process.env.IMAP_HOST || !process.env.IMAP_USER || !process.env.IMAP_PASS) {
     console.log('inbox poller disabled (IMAP_HOST / IMAP_USER / IMAP_PASS not set)')
@@ -206,4 +309,4 @@ async function startInboxPoller() {
   console.log('inbox poller started (this server is primary)')
 }
 
-export { startInboxPoller, pollInbox }
+export { startInboxPoller, pollInbox, listInboxMail, ingestInboxMail }