diff --git a/controller/latest.js b/controller/latest.js index 6e92105..56a1592 100644 --- a/controller/latest.js +++ b/controller/latest.js @@ -1,8 +1,36 @@ const { PrismaClient } = require('@prisma/client') -const { SitemapStream, streamToPromise } = require('sitemap') const redis = require('../utils/redis.js') const prisma = new PrismaClient() +function createSitemapXML(urls) { + const xml = urls.map(url => ` + + ${url} + never + 0.7 + + `).join(''); + + return ` + + ${xml} + `; +} + +function createSitemapIndexXML(sitemaps) { + const xml = sitemaps.map((sitemap, index) => ` + + https://api.preservetube.com/${sitemap} + ${new Date().toISOString()} + + `).join(''); + + return ` + + ${xml} + `; +} + exports.getLatest = async (req, res) => { let json const cached = await redis.get('latest') @@ -36,10 +64,10 @@ exports.getLatest = async (req, res) => { } exports.getSitemap = async (req, res) => { - const cachedSitemap = await redis.get('sitemap'); - if (cachedSitemap) { + const cachedSitemapIndex = await redis.get('sitemap-index'); + if (cachedSitemapIndex) { res.header('Content-Type', 'application/xml'); - return res.send(cachedSitemap); + return res.send(cachedSitemapIndex); } const dbVideos = await prisma.videos.findMany({ @@ -48,19 +76,27 @@ exports.getSitemap = async (req, res) => { }, }); - const videos = dbVideos.map((video) => ({ - url: `/videos/${video.id}`, - changefreq: 'never', - priority: 0.7, - })); + const urls = dbVideos.map((video) => `https://preservetube.com/watch?v=${video.id}`); + const sitemaps = []; + for (let i = 0; i < urls.length; i += 50000) { + const batch = urls.slice(i, i + 50000); + await redis.set(`sitemap-${sitemaps.length}`, createSitemapXML(batch), 'EX', 86400); + sitemaps.push(`sitemap-${sitemaps.length}.xml`); + } - const smStream = new SitemapStream({ hostname: 'https://preservetube.com' }); - videos.forEach((video) => smStream.write(video)); - smStream.end(); - - const sitemap = await streamToPromise(smStream).then((data) => data.toString()); - await redis.set('sitemap', sitemap, 'EX', 86400); + const sitemapIndexXML = createSitemapIndexXML(sitemaps); + await redis.set('sitemap-index', sitemapIndexXML, 'EX', 86400); res.header('Content-Type', 'application/xml'); - res.send(sitemap); + res.send(sitemapIndexXML); +}; + +exports.getSubSitemap = async (req, res) => { + const cachedSitemap = await redis.get(`sitemap-${req.params.index}`); + if (cachedSitemap) { + res.header('Content-Type', 'application/xml'); + return res.send(cachedSitemap); + } + + res.status(404).send(''); }; \ No newline at end of file diff --git a/index.js b/index.js index e93e56b..f1514e3 100644 --- a/index.js +++ b/index.js @@ -17,7 +17,8 @@ require('express-ws')(app) app.use(cors()) app.get('/latest', latestController.getLatest) -app.get('/sitemap.xml', latestController.getSitemap) +app.get('/sitemap-index.xml', latestController.getSitemap) +app.get('/sitemap-:index.xml', latestController.getSubSitemap) app.get('/video/:id', videoController.getVideo) app.get('/channel/:id', videoController.getChannel) diff --git a/package.json b/package.json index 0725cff..8d3fb65 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,6 @@ "node-fetch": "2", "readable-to-ms": "^1.0.3", "rolling-rate-limiter": "^0.4.2", - "sitemap": "^8.0.0", "wget-improved": "^3.4.0", "winston": "^3.8.2", "ws": "^8.17.1" diff --git a/yarn.lock b/yarn.lock index 7828fba..fed345b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -114,30 +114,11 @@ dependencies: "@types/trusted-types" "*" -"@types/node@*": - version "22.5.4" - resolved "https://registry.yarnpkg.com/@types/node/-/node-22.5.4.tgz#83f7d1f65bc2ed223bdbf57c7884f1d5a4fa84e8" - integrity sha512-FDuKUJQm/ju9fT/SeX/6+gBzoPzlVCzfzmGkwKvRHQVxi4BntVbyIwf6a4Xn62mrvndLiml6z/UBXIdEVjQLXg== - dependencies: - undici-types "~6.19.2" - "@types/node@^10.0.3": version "10.17.60" resolved "https://registry.yarnpkg.com/@types/node/-/node-10.17.60.tgz#35f3d6213daed95da7f0f73e75bcc6980e90597b" integrity sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw== -"@types/node@^17.0.5": - version "17.0.45" - resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.45.tgz#2c0fafd78705e7a18b7906b5201a522719dc5190" - integrity sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw== - -"@types/sax@^1.2.1": - version "1.2.7" - resolved "https://registry.yarnpkg.com/@types/sax/-/sax-1.2.7.tgz#ba5fe7df9aa9c89b6dff7688a19023dd2963091d" - integrity sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A== - dependencies: - "@types/node" "*" - "@types/stack-trace@^0.0.29": version "0.0.29" resolved "https://registry.yarnpkg.com/@types/stack-trace/-/stack-trace-0.0.29.tgz#eb7a7c60098edb35630ed900742a5ecb20cfcb4d" @@ -191,11 +172,6 @@ agent-base@6: dependencies: debug "4" -arg@^5.0.0: - version "5.0.2" - resolved "https://registry.yarnpkg.com/arg/-/arg-5.0.2.tgz#c81433cc427c92c4dcf4865142dbca6f15acd59c" - integrity sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg== - array-flatten@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2" @@ -1223,11 +1199,6 @@ sax@>=0.6.0: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== -sax@^1.2.4: - version "1.4.1" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" - integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== - saxes@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/saxes/-/saxes-6.0.0.tgz#fe5b4a4768df4f14a201b1ba6a65c1f3d9988cc5" @@ -1285,16 +1256,6 @@ simple-swizzle@^0.2.2: dependencies: is-arrayish "^0.3.1" -sitemap@^8.0.0: - version "8.0.0" - resolved "https://registry.yarnpkg.com/sitemap/-/sitemap-8.0.0.tgz#eb6ea48f95787cd680b83683c555d6f6b5a903fd" - integrity sha512-+AbdxhM9kJsHtruUF39bwS/B0Fytw6Fr1o4ZAIAEqA6cke2xcoO2GleBw9Zw7nRzILVEgz7zBM5GiTJjie1G9A== - dependencies: - "@types/node" "^17.0.5" - "@types/sax" "^1.2.1" - arg "^5.0.0" - sax "^1.2.4" - source-map@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" @@ -1389,11 +1350,6 @@ typedarray@^0.0.6: resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA== -undici-types@~6.19.2: - version "6.19.8" - resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02" - integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw== - universalify@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.2.0.tgz#6451760566fa857534745ab1dde952d1b1761be0"