From 16724a580f8b9d9d5bb51884bf2140237e8e2212 Mon Sep 17 00:00:00 2001 From: localhost Date: Sun, 22 Mar 2026 11:58:18 +0100 Subject: [PATCH] disallow rules for gptbot and bing --- public/robots.txt | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/public/robots.txt b/public/robots.txt index 1e10f2b..5002c19 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -68,9 +68,27 @@ Disallow: / # the crawling rate is simply insane User-agent: Bingbot +Disallow: /playlist +Disallow: /hashtag/* +Disallow: /live/* +Disallow: /user/* +Disallow: /shorts/* +Disallow: /c/* +Disallow: /@* +Disallow: /search +Disallow: /search?search=* Crawl-delay: 2 User-agent: GPTBot +Disallow: /playlist +Disallow: /hashtag/* +Disallow: /live/* +Disallow: /user/* +Disallow: /shorts/* +Disallow: /c/* +Disallow: /@* +Disallow: /search +Disallow: /search?search=* Crawl-delay: 2