# Kevmor Trade Supplies - robots.txt
# Updated: 2026-01-26
# Intent:
# 1) Allow AI SEARCH/assistants for AI SEO (citations)
# 2) Block AI TRAINING/dataset crawlers
# 3) Block heavy SEO/scrape tools
# 4) Keep PrestaShop system, account/checkout, modules, and noisy parameters out of crawl

# -----------------------------------
# AI SEARCH / ASSISTANTS (ALLOW + SAFE)
# -----------------------------------
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: DuckAssistBot
Allow: /

# PrestaShop system directories (note: /controllers/ is a folder, not the controller= parameter)
Disallow: /app/
Disallow: /cache/
Disallow: /classes/
Disallow: /config/
Disallow: /controllers/
Disallow: /download/
Disallow: /localization/
Disallow: /log/
Disallow: /mails/
Disallow: /override/
Disallow: /pdf/
Disallow: /src/
Disallow: /tools/
Disallow: /translations/
Disallow: /upload/
Disallow: /var/
Disallow: /vendor/
Disallow: /webservice/

# Account / cart / checkout / internal utility pages
Disallow: /authentication
Disallow: /password
Disallow: /password-recovery
Disallow: /address
Disallow: /addresses
Disallow: /cart
Disallow: /order
Disallow: /order-opc
Disallow: /order-history
Disallow: /history
Disallow: /order-follow
Disallow: /order-slip
Disallow: /identity
Disallow: /my-account
Disallow: /guest-tracking
Disallow: /my-shopping-carts
Disallow: /search
Disallow: /quick-order
Disallow: /discount
Disallow: /page-not-found
Disallow: /pagenotfound

# PrestaShop controller routes (SAFER: only known bad controllers)
Disallow: /*controller=address
Disallow: /*controller=addresses
Disallow: /*controller=authentication
Disallow: /*controller=cart
Disallow: /*controller=discount
Disallow: /*controller=history
Disallow: /*controller=identity
Disallow: /*controller=my-account
Disallow: /*controller=order
Disallow: /*controller=order-opc
Disallow: /*controller=order-slip
Disallow: /*controller=order-detail
Disallow: /*controller=order-follow
Disallow: /*controller=order-return
Disallow: /*controller=order-confirmation
Disallow: /*controller=pdf-invoice
Disallow: /*controller=pdf-order-return
Disallow: /*controller=pdf-order-slip
Disallow: /*controller=guest-tracking
Disallow: /*controller=statistics
Disallow: /*controller=search
Disallow: /*controller=password
Disallow: /*controller=module-

# Noisy / duplicate parameters
Disallow: /*?order=
Disallow: /*?tag=
Disallow: /*?id_currency=
Disallow: /*?search_query=
Disallow: /*?back=
Disallow: /*?n=
Disallow: /*&order=
Disallow: /*&tag=
Disallow: /*&id_currency=
Disallow: /*&search_query=
Disallow: /*&back=
Disallow: /*&n=

# Reduce crawl noise
Disallow: /ajax*.php

# -----------------------------
# AI TRAINING / DATASET (BLOCK)
# -----------------------------
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: Bytespider
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
Disallow: /

# -----------------------------
# HEAVY SEO / SCRAPE TOOLS (BLOCK)
# -----------------------------
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: DataForSeoBot
Disallow: /

# -----------------------------
# DEFAULT RULES (ALL OTHER BOTS)
# -----------------------------
User-agent: *
Allow: /

# PrestaShop system directories
Disallow: /app/
Disallow: /cache/
Disallow: /classes/
Disallow: /config/
Disallow: /controllers/
Disallow: /download/
Disallow: /localization/
Disallow: /log/
Disallow: /mails/
Disallow: /override/
Disallow: /pdf/
Disallow: /src/
Disallow: /tools/
Disallow: /translations/
Disallow: /upload/
Disallow: /var/
Disallow: /vendor/
Disallow: /webservice/

# Account / cart / checkout / internal utility pages
Disallow: /authentication
Disallow: /password
Disallow: /password-recovery
Disallow: /address
Disallow: /addresses
Disallow: /cart
Disallow: /order
Disallow: /order-opc
Disallow: /order-history
Disallow: /history
Disallow: /order-follow
Disallow: /order-slip
Disallow: /identity
Disallow: /my-account
Disallow: /guest-tracking
Disallow: /my-shopping-carts
Disallow: /search
Disallow: /quick-order
Disallow: /discount
Disallow: /page-not-found
Disallow: /pagenotfound

# PrestaShop controller routes (SAFER: only known bad controllers)
Disallow: /*controller=address
Disallow: /*controller=addresses
Disallow: /*controller=authentication
Disallow: /*controller=cart
Disallow: /*controller=discount
Disallow: /*controller=history
Disallow: /*controller=identity
Disallow: /*controller=my-account
Disallow: /*controller=order
Disallow: /*controller=order-opc
Disallow: /*controller=order-slip
Disallow: /*controller=order-detail
Disallow: /*controller=order-follow
Disallow: /*controller=order-return
Disallow: /*controller=order-confirmation
Disallow: /*controller=pdf-invoice
Disallow: /*controller=pdf-order-return
Disallow: /*controller=pdf-order-slip
Disallow: /*controller=guest-tracking
Disallow: /*controller=statistics
Disallow: /*controller=search
Disallow: /*controller=password
Disallow: /*controller=module-

# Noisy / duplicate parameters
Disallow: /*?order=
Disallow: /*?tag=
Disallow: /*?id_currency=
Disallow: /*?search_query=
Disallow: /*?back=
Disallow: /*?n=
Disallow: /*&order=
Disallow: /*&tag=
Disallow: /*&id_currency=
Disallow: /*&search_query=
Disallow: /*&back=
Disallow: /*&n=

# Reduce crawl noise
Disallow: /ajax*.php

# Sitemaps
Sitemap: https://kevmor.com.au/1_index_sitemap.xml
Sitemap: https://kevmor.com.au/1_en_0_sitemap.xml