# global robots.txt file to cover all of our sites
# this is configured in /etc/apache2/apache2.conf
#
# If the Joomla site is installed within a folder such as at
# e.g. www.example.com/joomla/ the joomla folder name MUST be prefixed to the disallowed
# path, e.g. the Disallow rule for the /administrator/ folder
# MUST be changed to read Disallow: /joomla/administrator/
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/orig.html
#
# For syntax checking, see:
# http://tool.motoricerca.info/robots-checker.phtml

User-agent: *
Crawl-delay: 30
Disallow: /administrator/
Disallow: /bin/
Disallow: /cache/
Disallow: /cli/
Disallow: /components/
Disallow: /images/
Disallow: /includes/
Disallow: /installation/
Disallow: /language/
Disallow: /layouts/
Disallow: /libraries/
Disallow: /logs/
Disallow: /media/
Disallow: /modules/
Disallow: /plugins/
Disallow: /templates/
Disallow: /tmp/
#these groups do not exist but are queried by bots
Disallow: /groups/2
Disallow: /groups/5
Disallow: /groups/7
Disallow: /groups/8
Disallow: /groups/9
Disallow: /groups/10
Disallow: /groups/11
Disallow: /groups/13
Disallow: /groups/15
Disallow: /groups/18
Disallow: /groups/19
Disallow: /groups/20
Disallow: /groups/22
Disallow: /groups/23
Disallow: /groups/24
Disallow: /groups/25
Disallow: /groups/26
Disallow: /groups/27
Disallow: /groups/28

User-agent: BLEXBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-Agent: SputnikBot
Disallow:/
#User-agent: Baiduspider
#Disallow: /
User-agent: SeznamBot
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by VenTek (info@ventek.ch)
Disallow: /
User-agent: SEOkicks-Robot
Disallow: /
user-agent: AhrefsBot
disallow: /
User-agent: spbot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: dotbot
Disallow: /
User-agent: The Knowledge AI
Disallow: /
User-agent: MauiBot
Disallow: /