66 lines
1.1 KiB
Plaintext
66 lines
1.1 KiB
Plaintext
User-agent: *
|
|
Disallow: /playlist
|
|
Disallow: /hashtag/*
|
|
Disallow: /live/*
|
|
Disallow: /user/*
|
|
Disallow: /shorts/*
|
|
Disallow: /c/*
|
|
Disallow: /@*
|
|
Disallow: /search
|
|
Disallow: /search?search=*
|
|
|
|
# seo backlink bots are a waste of bandwidth
|
|
User-agent: dotbot
|
|
Disallow: /
|
|
|
|
User-agent: BLEXBot
|
|
Disallow: /
|
|
|
|
User-agent: SemrushBot
|
|
Disallow: /
|
|
|
|
User-agent: AhrefsBot
|
|
Disallow: /
|
|
|
|
User-agent: barkrowler
|
|
Disallow: /
|
|
|
|
User-agent: VelenPublicWebCrawler
|
|
Disallow: /
|
|
|
|
User-agent: AwarioRssBot
|
|
Disallow: /
|
|
|
|
User-agent: AwarioSmartBot
|
|
Disallow: /
|
|
|
|
User-agent: PetalBot
|
|
Disallow: /
|
|
|
|
User-Agent: ImagesiftBot
|
|
Disallow: /
|
|
|
|
User-agent: DataForSeoBot
|
|
Disallow: /
|
|
|
|
User-agent: facebookexternalhit
|
|
Disallow: /
|
|
|
|
# ai crawler for alexa... who uses alexa.
|
|
User-agent: Amazonbot
|
|
Disallow: /
|
|
|
|
# apple does not know how to behave.
|
|
User-agent: Applebot
|
|
Disallow: /
|
|
|
|
User-agent: Applebot-Extended
|
|
Disallow: /
|
|
|
|
# the crawling rate is simply insane
|
|
User-agent: Bingbot
|
|
Crawl-delay: 2
|
|
|
|
# this is intentionally only accessable for search engine crawlers.
|
|
# do you want to crawl too? get in touch https://preservetube.com/about
|
|
Sitemap: https://api.preservetube.com/sitemap.xml |