diff --git a/public/robots.txt b/public/robots.txt index 1e10f2b..5002c19 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -68,9 +68,27 @@ Disallow: / # the crawling rate is simply insane User-agent: Bingbot +Disallow: /playlist +Disallow: /hashtag/* +Disallow: /live/* +Disallow: /user/* +Disallow: /shorts/* +Disallow: /c/* +Disallow: /@* +Disallow: /search +Disallow: /search?search=* Crawl-delay: 2 User-agent: GPTBot +Disallow: /playlist +Disallow: /hashtag/* +Disallow: /live/* +Disallow: /user/* +Disallow: /shorts/* +Disallow: /c/* +Disallow: /@* +Disallow: /search +Disallow: /search?search=* Crawl-delay: 2