41 lines
606 B
Plaintext
41 lines
606 B
Plaintext
User-agent: *
|
|
Disallow: /playlist
|
|
Disallow: /hashtag/*
|
|
Disallow: /live/*
|
|
Disallow: /user/*
|
|
Disallow: /shorts/*
|
|
Disallow: /c/*
|
|
Disallow: /@*
|
|
|
|
# seo backlink bots are a waste of bandwidth
|
|
User-agent: dotbot
|
|
Disallow: /
|
|
|
|
User-agent: BLEXBot
|
|
Disallow: /
|
|
|
|
User-agent: SemrushBot
|
|
Disallow: /
|
|
|
|
User-agent: AhrefsBot
|
|
Disallow: /
|
|
|
|
User-agent: barkrowler
|
|
Disallow: /
|
|
|
|
User-agent: VelenPublicWebCrawler
|
|
Disallow: /
|
|
|
|
User-agent: AwarioRssBot
|
|
Disallow: /
|
|
|
|
User-agent: AwarioSmartBot
|
|
Disallow: /
|
|
|
|
User-agent: PetalBot
|
|
Disallow: /
|
|
|
|
User-Agent: ImagesiftBot
|
|
Disallow: /
|
|
|
|
Sitemap: https://api.preservetube.com/sitemap.xml |