User-agent: *
Disallow: /playlist
Disallow: /hashtag/*
Disallow: /live/*
Disallow: /user/*
Disallow: /shorts/*
Disallow: /c/*
Disallow: /@*
Disallow: /search
Disallow: /search?search=*

# seo backlink bots are a waste of bandwidth
User-agent: dotbot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: barkrowler
Disallow: /

User-agent: VelenPublicWebCrawler
Disallow: /

User-agent: AwarioRssBot
Disallow: /

User-agent: AwarioSmartBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-Agent: ImagesiftBot 
Disallow: / 

User-agent: DataForSeoBot
Disallow: /

User-agent: facebookexternalhit
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-Agent: MJ12bot
Disallow: /

User-agent: Owler
Disallow: /

# ai crawler for alexa... who uses alexa.
User-agent: Amazonbot
Disallow: /

# apple does not know how to behave.
User-agent: Applebot
Disallow: /

User-agent: Applebot-Extended  
Disallow: /

# the crawling rate is simply insane
User-agent: Bingbot
Crawl-delay: 2

# this is intentionally only accessable for search engine crawlers. 
# do you want to crawl too? get in touch https://preservetube.com/about
Sitemap: https://api.preservetube.com/sitemap.xml