User-Agent: *
Crawl-delay: 2
Disallow: /cdn-cgi/

Sitemap: https://www.expofit.nl/media/expofit_sitemap_category.xml
Sitemap: https://www.expofit.nl/media/expofit_sitemap_cms.xml
Sitemap: https://www.expofit.nl/media/expofit_sitemap_product.xml

# MageHost: prevent Google from going wild on filter/sorting params
Disallow: *?*%20*
Disallow: *?*%25*
Disallow: *?*%2C*
Disallow: *?*,*
Disallow: *?*%5B*
Disallow: *?limit=*
Disallow: *&limit=*
Disallow: *?order=*
Disallow: *&order=*
Disallow: *?dir=*
Disallow: *&dir=*
Disallow: *?mode=*
Disallow: *&mode=*
Disallow: *?___store=*
Disallow: *&___store=*
Disallow: *?___from_store=*
Disallow: *&___from_store=*
Disallow: *?SID=*
Disallow: *&SID=*
Disallow: *?___SID=*
Disallow: *&___SID=*
Disallow: */zoeken/
Disallow: */winkelwagen$
Disallow: */checkout
Disallow: */mijn-account/

# =========================
# CORE SEARCH ENGINE BOTS
# =========================
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: Applebot
Allow: /

User-agent: PetalBot
Allow: /

# =========================
# AI / LLM CRAWLERS
# =========================
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: CCBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# =========================
# SEO / ANALYSIS CRAWLERS
# =========================
User-agent: SemrushBot
Allow: /

User-agent: SemrushBot-SA
Allow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# =========================
# ADVERTISING BOTS
# =========================
User-agent: AdsBot-Google
Allow: /

User-agent: Mediapartners-Google
Allow: /

User-agent: Bytespider
Disallow: /