User-agent: *
Crawl-delay: 30
Allow: /

Sitemap: https://www.audioaffair.co.uk/sitemap.xml

# Google Image Crawler Setup
User-agent: Googlebot
Allow: /
User-agent: Googlebot-image
Allow: /
User-agent: Googlebot-Mobile
Allow: /

## Added by Host
User-agent: Rogerbot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: TwengaBot
Disallow: /
User-agent: 008
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: WotBox
Disallow: /
User-agent: Sosospider
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: ZumBot
Disallow: /
User-agent: YandexBot
Disallow: /
User-agent: MegaIndex
Disallow: /
User-agent: Pcore
Disallow: /
User-agent: The Knowledge AI
Disallow: /

# Do not index the shared files Magento
Disallow: /api.php
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /cleanup.php
Disallow: /apc.php
Disallow: /memcache.php
Disallow: /phpinfo.php
Disallow: /error_log
Disallow: /get.php
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /README.txt
Disallow: /RELEASE_NOTES.txt

# Do not index the page checkout and user account
Disallow: /checkout/
Disallow: /onestepcheckout/
Disallow: /customer/
Disallow: /customer/account/
Disallow: /customer/account/login/

# Search Filters and product comparison
Disallow: /catalog/product_compare/
Disallow: /catalogsearch/
Disallow: /Shopby/
Disallow: /*price=
Disallow: /*sizefilter=
Disallow: /*colour=