2025-10-14 18:53:40 +00:00
|
|
|
User-agent: *
|
|
|
|
|
Disallow: /playlist
|
|
|
|
|
Disallow: /hashtag/*
|
|
|
|
|
Disallow: /live/*
|
|
|
|
|
Disallow: /user/*
|
|
|
|
|
Disallow: /shorts/*
|
|
|
|
|
Disallow: /c/*
|
|
|
|
|
Disallow: /@*
|
2026-01-31 21:05:10 +00:00
|
|
|
Disallow: /search
|
|
|
|
|
Disallow: /search?search=*
|
2025-10-14 18:53:40 +00:00
|
|
|
|
|
|
|
|
# seo backlink bots are a waste of bandwidth
|
|
|
|
|
User-agent: dotbot
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
|
|
|
|
User-agent: BLEXBot
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
|
|
|
|
User-agent: SemrushBot
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
|
|
|
|
User-agent: AhrefsBot
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
|
|
|
|
User-agent: barkrowler
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
|
|
|
|
User-agent: VelenPublicWebCrawler
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
|
|
|
|
User-agent: AwarioRssBot
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
|
|
|
|
User-agent: AwarioSmartBot
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
|
|
|
|
User-agent: PetalBot
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
|
|
|
|
User-Agent: ImagesiftBot
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
2026-02-01 15:17:37 +00:00
|
|
|
User-agent: DataForSeoBot
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
|
|
|
|
User-agent: facebookexternalhit
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
2026-02-16 12:34:22 +00:00
|
|
|
User-agent: meta-externalagent
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
2026-02-01 15:19:20 +00:00
|
|
|
User-Agent: MJ12bot
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
|
|
|
|
User-agent: Owler
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
2026-01-31 17:19:39 +00:00
|
|
|
# ai crawler for alexa... who uses alexa.
|
|
|
|
|
User-agent: Amazonbot
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
|
|
|
|
# apple does not know how to behave.
|
|
|
|
|
User-agent: Applebot
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
|
|
|
|
User-agent: Applebot-Extended
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
|
|
|
|
# the crawling rate is simply insane
|
|
|
|
|
User-agent: Bingbot
|
2026-03-22 10:58:18 +00:00
|
|
|
Disallow: /playlist
|
|
|
|
|
Disallow: /hashtag/*
|
|
|
|
|
Disallow: /live/*
|
|
|
|
|
Disallow: /user/*
|
|
|
|
|
Disallow: /shorts/*
|
|
|
|
|
Disallow: /c/*
|
|
|
|
|
Disallow: /@*
|
|
|
|
|
Disallow: /search
|
|
|
|
|
Disallow: /search?search=*
|
2026-01-31 17:19:39 +00:00
|
|
|
Crawl-delay: 2
|
|
|
|
|
|
2026-02-18 22:27:17 +00:00
|
|
|
User-agent: GPTBot
|
2026-03-22 10:58:18 +00:00
|
|
|
Disallow: /playlist
|
|
|
|
|
Disallow: /hashtag/*
|
|
|
|
|
Disallow: /live/*
|
|
|
|
|
Disallow: /user/*
|
|
|
|
|
Disallow: /shorts/*
|
|
|
|
|
Disallow: /c/*
|
|
|
|
|
Disallow: /@*
|
|
|
|
|
Disallow: /search
|
|
|
|
|
Disallow: /search?search=*
|
2026-02-18 22:27:17 +00:00
|
|
|
Crawl-delay: 2
|
|
|
|
|
|
|
|
|
|
|
2026-02-01 15:15:53 +00:00
|
|
|
# this is intentionally only accessable for search engine crawlers.
|
|
|
|
|
# do you want to crawl too? get in touch https://preservetube.com/about
|
2025-10-14 18:53:40 +00:00
|
|
|
Sitemap: https://api.preservetube.com/sitemap.xml
|
