preservetube
/
backend
mirror of https://github.com/preservetube/backend.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

disallow rules for gptbot and bing

This commit is contained in:
localhost 2026-03-22 11:58:18 +01:00
parent 466bad6c05
commit 16724a580f
1 changed files with 18 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
public/robots.txt
View File

@ -68,9 +68,27 @@ Disallow: /
# the crawling rate is simply insane
User-agent: Bingbot
Disallow: /playlist
Disallow: /hashtag/*
Disallow: /live/*
Disallow: /user/*
Disallow: /shorts/*
Disallow: /c/*
Disallow: /@*
Disallow: /search
Disallow: /search?search=*
Crawl-delay: 2
User-agent: GPTBot
Disallow: /playlist
Disallow: /hashtag/*
Disallow: /live/*
Disallow: /user/*
Disallow: /shorts/*
Disallow: /c/*
Disallow: /@*
Disallow: /search
Disallow: /search?search=*
Crawl-delay: 2