User-agent: *
Disallow: /playlist
Disallow: /hashtag/*
Disallow: /live/*
Disallow: /user/*
Disallow: /shorts/*
Disallow: /c/*
Disallow: /@*
Disallow: /search
Disallow: /search?search=*

# seo backlink bots are a waste of bandwidth
User-agent: dotbot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: barkrowler
Disallow: /

User-agent: VelenPublicWebCrawler
Disallow: /

User-agent: AwarioRssBot
Disallow: /

User-agent: AwarioSmartBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-Agent: ImagesiftBot 
Disallow: / 

# ai crawler for alexa... who uses alexa.
User-agent: Amazonbot
Disallow: /

# apple does not know how to behave.
User-agent: Applebot
Disallow: /

User-agent: Applebot-Extended  
Disallow: /

# the crawling rate is simply insane
User-agent: Bingbot
Crawl-delay: 2

Sitemap: https://api.preservetube.com/sitemap.xml