diff --git a/public/robots.txt b/public/robots.txt index eb81e15..ca7f02d 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -40,6 +40,12 @@ Disallow: / User-Agent: ImagesiftBot Disallow: / +User-agent: DataForSeoBot +Disallow: / + +User-agent: facebookexternalhit +Disallow: / + # ai crawler for alexa... who uses alexa. User-agent: Amazonbot Disallow: /