From 69619c3562f59f8b42e0d3c58289d5dce40f78e7 Mon Sep 17 00:00:00 2001 From: localhost Date: Thu, 12 Sep 2024 18:32:18 +0200 Subject: [PATCH] sitemap --- controller/latest.js | 33 ++++++++++++++++++++++++++++++++- index.js | 2 ++ package.json | 1 + yarn.lock | 44 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 1 deletion(-) diff --git a/controller/latest.js b/controller/latest.js index 3924c77..6e92105 100644 --- a/controller/latest.js +++ b/controller/latest.js @@ -1,4 +1,5 @@ const { PrismaClient } = require('@prisma/client') +const { SitemapStream, streamToPromise } = require('sitemap') const redis = require('../utils/redis.js') const prisma = new PrismaClient() @@ -32,4 +33,34 @@ exports.getLatest = async (req, res) => { } res.json(json) -} \ No newline at end of file +} + +exports.getSitemap = async (req, res) => { + const cachedSitemap = await redis.get('sitemap'); + if (cachedSitemap) { + res.header('Content-Type', 'application/xml'); + return res.send(cachedSitemap); + } + + const dbVideos = await prisma.videos.findMany({ + select: { + id: true, + }, + }); + + const videos = dbVideos.map((video) => ({ + url: `/videos/${video.id}`, + changefreq: 'never', + priority: 0.7, + })); + + const smStream = new SitemapStream({ hostname: 'https://preservetube.com' }); + videos.forEach((video) => smStream.write(video)); + smStream.end(); + + const sitemap = await streamToPromise(smStream).then((data) => data.toString()); + await redis.set('sitemap', sitemap, 'EX', 86400); + + res.header('Content-Type', 'application/xml'); + res.send(sitemap); +}; \ No newline at end of file diff --git a/index.js b/index.js index b60b2f0..e93e56b 100644 --- a/index.js +++ b/index.js @@ -17,6 +17,8 @@ require('express-ws')(app) app.use(cors()) app.get('/latest', latestController.getLatest) +app.get('/sitemap.xml', latestController.getSitemap) + app.get('/video/:id', videoController.getVideo) app.get('/channel/:id', videoController.getChannel) app.get('/channel/:id/videos', videoController.getOnlyChannelVideos) diff --git a/package.json b/package.json index 8d3fb65..0725cff 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "node-fetch": "2", "readable-to-ms": "^1.0.3", "rolling-rate-limiter": "^0.4.2", + "sitemap": "^8.0.0", "wget-improved": "^3.4.0", "winston": "^3.8.2", "ws": "^8.17.1" diff --git a/yarn.lock b/yarn.lock index fed345b..7828fba 100644 --- a/yarn.lock +++ b/yarn.lock @@ -114,11 +114,30 @@ dependencies: "@types/trusted-types" "*" +"@types/node@*": + version "22.5.4" + resolved "https://registry.yarnpkg.com/@types/node/-/node-22.5.4.tgz#83f7d1f65bc2ed223bdbf57c7884f1d5a4fa84e8" + integrity sha512-FDuKUJQm/ju9fT/SeX/6+gBzoPzlVCzfzmGkwKvRHQVxi4BntVbyIwf6a4Xn62mrvndLiml6z/UBXIdEVjQLXg== + dependencies: + undici-types "~6.19.2" + "@types/node@^10.0.3": version "10.17.60" resolved "https://registry.yarnpkg.com/@types/node/-/node-10.17.60.tgz#35f3d6213daed95da7f0f73e75bcc6980e90597b" integrity sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw== +"@types/node@^17.0.5": + version "17.0.45" + resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.45.tgz#2c0fafd78705e7a18b7906b5201a522719dc5190" + integrity sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw== + +"@types/sax@^1.2.1": + version "1.2.7" + resolved "https://registry.yarnpkg.com/@types/sax/-/sax-1.2.7.tgz#ba5fe7df9aa9c89b6dff7688a19023dd2963091d" + integrity sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A== + dependencies: + "@types/node" "*" + "@types/stack-trace@^0.0.29": version "0.0.29" resolved "https://registry.yarnpkg.com/@types/stack-trace/-/stack-trace-0.0.29.tgz#eb7a7c60098edb35630ed900742a5ecb20cfcb4d" @@ -172,6 +191,11 @@ agent-base@6: dependencies: debug "4" +arg@^5.0.0: + version "5.0.2" + resolved "https://registry.yarnpkg.com/arg/-/arg-5.0.2.tgz#c81433cc427c92c4dcf4865142dbca6f15acd59c" + integrity sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg== + array-flatten@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2" @@ -1199,6 +1223,11 @@ sax@>=0.6.0: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== +sax@^1.2.4: + version "1.4.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" + integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== + saxes@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/saxes/-/saxes-6.0.0.tgz#fe5b4a4768df4f14a201b1ba6a65c1f3d9988cc5" @@ -1256,6 +1285,16 @@ simple-swizzle@^0.2.2: dependencies: is-arrayish "^0.3.1" +sitemap@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/sitemap/-/sitemap-8.0.0.tgz#eb6ea48f95787cd680b83683c555d6f6b5a903fd" + integrity sha512-+AbdxhM9kJsHtruUF39bwS/B0Fytw6Fr1o4ZAIAEqA6cke2xcoO2GleBw9Zw7nRzILVEgz7zBM5GiTJjie1G9A== + dependencies: + "@types/node" "^17.0.5" + "@types/sax" "^1.2.1" + arg "^5.0.0" + sax "^1.2.4" + source-map@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" @@ -1350,6 +1389,11 @@ typedarray@^0.0.6: resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA== +undici-types@~6.19.2: + version "6.19.8" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02" + integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw== + universalify@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.2.0.tgz#6451760566fa857534745ab1dde952d1b1761be0"