56 lines
872 B
Plaintext
56 lines
872 B
Plaintext
User-agent: *
|
|
Disallow: /playlist
|
|
Disallow: /hashtag/*
|
|
Disallow: /live/*
|
|
Disallow: /user/*
|
|
Disallow: /shorts/*
|
|
Disallow: /c/*
|
|
Disallow: /@*
|
|
|
|
# seo backlink bots are a waste of bandwidth
|
|
User-agent: dotbot
|
|
Disallow: /
|
|
|
|
User-agent: BLEXBot
|
|
Disallow: /
|
|
|
|
User-agent: SemrushBot
|
|
Disallow: /
|
|
|
|
User-agent: AhrefsBot
|
|
Disallow: /
|
|
|
|
User-agent: barkrowler
|
|
Disallow: /
|
|
|
|
User-agent: VelenPublicWebCrawler
|
|
Disallow: /
|
|
|
|
User-agent: AwarioRssBot
|
|
Disallow: /
|
|
|
|
User-agent: AwarioSmartBot
|
|
Disallow: /
|
|
|
|
User-agent: PetalBot
|
|
Disallow: /
|
|
|
|
User-Agent: ImagesiftBot
|
|
Disallow: /
|
|
|
|
# ai crawler for alexa... who uses alexa.
|
|
User-agent: Amazonbot
|
|
Disallow: /
|
|
|
|
# apple does not know how to behave.
|
|
User-agent: Applebot
|
|
Disallow: /
|
|
|
|
User-agent: Applebot-Extended
|
|
Disallow: /
|
|
|
|
# the crawling rate is simply insane
|
|
User-agent: Bingbot
|
|
Crawl-delay: 2
|
|
|
|
Sitemap: https://api.preservetube.com/sitemap.xml |