moving sitemaps to built in stuff, fixing urls

controller/latest.js

@@ -1,8 +1,36 @@
 const { PrismaClient } =  require('@prisma/client')
-const { SitemapStream, streamToPromise } = require('sitemap')
 const redis = require('../utils/redis.js')
 const prisma = new PrismaClient()
 
+function createSitemapXML(urls) {
+    const xml = urls.map(url => `
+        <url>
+            <loc>${url}</loc>
+            <changefreq>never</changefreq>
+            <priority>0.7</priority>
+        </url>
+    `).join('');
+
+    return `<?xml version="1.0" encoding="UTF-8"?>
+    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        ${xml}
+    </urlset>`;
+}
+
+function createSitemapIndexXML(sitemaps) {
+    const xml = sitemaps.map((sitemap, index) => `
+        <sitemap>
+            <loc>https://api.preservetube.com/${sitemap}</loc>
+            <lastmod>${new Date().toISOString()}</lastmod>
+        </sitemap>
+    `).join('');
+
+    return `<?xml version="1.0" encoding="UTF-8"?>
+    <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        ${xml}
+    </sitemapindex>`;
+}
+
 exports.getLatest = async (req, res) => {
     let json
     const cached = await redis.get('latest')
@@ -36,10 +64,10 @@ exports.getLatest = async (req, res) => {
 }
 
 exports.getSitemap = async (req, res) => {
-    const cachedSitemap = await redis.get('sitemap');
-    if (cachedSitemap) {
+    const cachedSitemapIndex = await redis.get('sitemap-index');
+    if (cachedSitemapIndex) {
         res.header('Content-Type', 'application/xml');
-        return res.send(cachedSitemap);
+        return res.send(cachedSitemapIndex);
     }
 
     const dbVideos = await prisma.videos.findMany({
@@ -48,19 +76,27 @@ exports.getSitemap = async (req, res) => {
         },
     });
 
-    const videos = dbVideos.map((video) => ({
-        url: `/videos/${video.id}`,
-        changefreq: 'never',
-        priority: 0.7,
-    }));
+    const urls = dbVideos.map((video) => `https://preservetube.com/watch?v=${video.id}`);
+    const sitemaps = [];
+    for (let i = 0; i < urls.length; i += 50000) {
+        const batch = urls.slice(i, i + 50000);
+        await redis.set(`sitemap-${sitemaps.length}`, createSitemapXML(batch), 'EX', 86400);
+        sitemaps.push(`sitemap-${sitemaps.length}.xml`);
+    }
 
-    const smStream = new SitemapStream({ hostname: 'https://preservetube.com' });
-    videos.forEach((video) => smStream.write(video));
-    smStream.end();
-
-    const sitemap = await streamToPromise(smStream).then((data) => data.toString());
-    await redis.set('sitemap', sitemap, 'EX', 86400);
+    const sitemapIndexXML = createSitemapIndexXML(sitemaps);
+    await redis.set('sitemap-index', sitemapIndexXML, 'EX', 86400);
 
     res.header('Content-Type', 'application/xml');
-    res.send(sitemap);
+    res.send(sitemapIndexXML);
+};
+
+exports.getSubSitemap = async (req, res) => {
+    const cachedSitemap = await redis.get(`sitemap-${req.params.index}`);
+    if (cachedSitemap) {
+        res.header('Content-Type', 'application/xml');
+        return res.send(cachedSitemap);
+    }
+
+    res.status(404).send('');
 };

index.js

@@ -17,7 +17,8 @@ require('express-ws')(app)
 app.use(cors())
 
 app.get('/latest', latestController.getLatest)
-app.get('/sitemap.xml', latestController.getSitemap)
+app.get('/sitemap-index.xml', latestController.getSitemap)
+app.get('/sitemap-:index.xml', latestController.getSubSitemap)
 
 app.get('/video/:id', videoController.getVideo)
 app.get('/channel/:id', videoController.getChannel)

package.json

@@ -22,7 +22,6 @@
     "node-fetch": "2",
     "readable-to-ms": "^1.0.3",
     "rolling-rate-limiter": "^0.4.2",
-    "sitemap": "^8.0.0",
     "wget-improved": "^3.4.0",
     "winston": "^3.8.2",
     "ws": "^8.17.1"

yarn.lock

@@ -114,30 +114,11 @@
   dependencies:
     "@types/trusted-types" "*"
 
-"@types/node@*":
-  version "22.5.4"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-22.5.4.tgz#83f7d1f65bc2ed223bdbf57c7884f1d5a4fa84e8"
-  integrity sha512-FDuKUJQm/ju9fT/SeX/6+gBzoPzlVCzfzmGkwKvRHQVxi4BntVbyIwf6a4Xn62mrvndLiml6z/UBXIdEVjQLXg==
-  dependencies:
-    undici-types "~6.19.2"
-
 "@types/node@^10.0.3":
   version "10.17.60"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-10.17.60.tgz#35f3d6213daed95da7f0f73e75bcc6980e90597b"
   integrity sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==
 
-"@types/node@^17.0.5":
-  version "17.0.45"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.45.tgz#2c0fafd78705e7a18b7906b5201a522719dc5190"
-  integrity sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==
-
-"@types/sax@^1.2.1":
-  version "1.2.7"
-  resolved "https://registry.yarnpkg.com/@types/sax/-/sax-1.2.7.tgz#ba5fe7df9aa9c89b6dff7688a19023dd2963091d"
-  integrity sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==
-  dependencies:
-    "@types/node" "*"
-
 "@types/stack-trace@^0.0.29":
   version "0.0.29"
   resolved "https://registry.yarnpkg.com/@types/stack-trace/-/stack-trace-0.0.29.tgz#eb7a7c60098edb35630ed900742a5ecb20cfcb4d"
@@ -191,11 +172,6 @@ agent-base@6:
   dependencies:
     debug "4"
 
-arg@^5.0.0:
-  version "5.0.2"
-  resolved "https://registry.yarnpkg.com/arg/-/arg-5.0.2.tgz#c81433cc427c92c4dcf4865142dbca6f15acd59c"
-  integrity sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==
-
 array-flatten@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2"
@@ -1223,11 +1199,6 @@ sax@>=0.6.0:
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
   integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
 
-sax@^1.2.4:
-  version "1.4.1"
-  resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f"
-  integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==
-
 saxes@^6.0.0:
   version "6.0.0"
   resolved "https://registry.yarnpkg.com/saxes/-/saxes-6.0.0.tgz#fe5b4a4768df4f14a201b1ba6a65c1f3d9988cc5"
@@ -1285,16 +1256,6 @@ simple-swizzle@^0.2.2:
   dependencies:
     is-arrayish "^0.3.1"
 
-sitemap@^8.0.0:
-  version "8.0.0"
-  resolved "https://registry.yarnpkg.com/sitemap/-/sitemap-8.0.0.tgz#eb6ea48f95787cd680b83683c555d6f6b5a903fd"
-  integrity sha512-+AbdxhM9kJsHtruUF39bwS/B0Fytw6Fr1o4ZAIAEqA6cke2xcoO2GleBw9Zw7nRzILVEgz7zBM5GiTJjie1G9A==
-  dependencies:
-    "@types/node" "^17.0.5"
-    "@types/sax" "^1.2.1"
-    arg "^5.0.0"
-    sax "^1.2.4"
-
 source-map@~0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
@@ -1389,11 +1350,6 @@ typedarray@^0.0.6:
   resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777"
   integrity sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==
 
-undici-types@~6.19.2:
-  version "6.19.8"
-  resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02"
-  integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==
-
 universalify@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.2.0.tgz#6451760566fa857534745ab1dde952d1b1761be0"