# =============================================================================
# robots.txt — repairmate.com.au
# Policy: allow standard search engines; curate AI/LLM crawlers via an explicit
# allow-list; default-deny AI input & training for every other crawler.
# Content-Signal follows the Content Signals Policy (search / ai-input / ai-train).
# =============================================================================

# ---------- Search engines ----------
User-agent: Googlebot
Allow: /
# Faceted-nav crawl block (Task 1a — infinite filter permutations under /shop)
Disallow: /shop/*?*q=
Disallow: /shop/*?*order=
Disallow: /shop/*?*shop_view=
Disallow: /shop/*?*resultsPerPage=
# Legacy dead schemes (Task 3 A/C)
Disallow: /shop/shop/
Disallow: /*filter_*

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Crawl-delay: 5
Allow: /
# Faceted-nav crawl block (Task 1a — infinite filter permutations under /shop)
Disallow: /shop/*?*q=
Disallow: /shop/*?*order=
Disallow: /shop/*?*shop_view=
Disallow: /shop/*?*resultsPerPage=
# Legacy dead schemes (Task 3 A/C)
Disallow: /shop/shop/
Disallow: /*filter_*

User-agent: DuckDuckBot
Allow: /

User-agent: YandexBot
Crawl-delay: 10
Allow: /

User-agent: Baiduspider
Crawl-delay: 10
Allow: /

User-agent: Applebot
Allow: /

# ---------- AI / LLM crawlers — explicitly ALLOWED ----------
# Full access: discoverable in search, usable as AI input, and for AI training.

# Allow ChatGPT (OpenAI)
User-agent: GPTBot
Disallow:
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Allow Claude (Anthropic)
User-agent: ClaudeBot
Disallow:
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Allow Gemini (Google's AI crawler)
User-agent: Google-Extended
Disallow:
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Allow Perplexity AI
User-agent: PerplexityBot
Disallow:
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Allow Qwen (Alibaba)
User-agent: QwenBot
Disallow:
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Allow Apple Intelligence
User-agent: Applebot-Extended
Disallow:
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Allow Meta AI
User-agent: Meta-ExternalAgent
Disallow:
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Allow Cohere AI
User-agent: cohere-ai
Disallow:
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# ---------- SEO crawlers ----------
User-agent: AhrefsBot
Crawl-delay: 10
Allow: /

User-agent: SemrushBot
Crawl-delay: 10
Allow: /

User-agent: MJ12bot
Crawl-delay: 10
Allow: /

# ---------- Default — all other crawlers ----------
# Crawlable for search, but NOT permitted as AI input or for AI training.
# This is the fallback that blocks all other AI crawlers by default.
User-agent: *
Allow: /
Disallow: /account/
Disallow: /checkout/
Disallow: /order-details/
Disallow: /login/
Disallow: /register/
Disallow: /track-order/
Disallow: /review/
Content-Signal: search=yes, ai-input=no, ai-train=no
# Faceted-nav crawl block (Task 1a — infinite filter permutations under /shop)
Disallow: /shop/*?*q=
Disallow: /shop/*?*order=
Disallow: /shop/*?*shop_view=
Disallow: /shop/*?*resultsPerPage=
# Legacy dead schemes (Task 3 A/C)
Disallow: /shop/shop/
Disallow: /*filter_*

# ---------- Sitemaps ----------
Sitemap: https://repairmate.com.au/sitemap.xml
Sitemap: https://repairmate.com.au/sitemap-news.xml
Sitemap: https://repairmate.com.au/sitemap-images.xml
Sitemap: https://repairmate.com.au/shop/1_index_sitemap.xml

# ---------- AI / LLM index (non-XML, referenced for discovery) ----------
# https://llmstxt.org/ — curated markdown index for LLM ingestion
# Located at: https://repairmate.com.au/llms.txt
# Full content version at: https://repairmate.com.au/llms-full.txt
# AI training preferences: https://repairmate.com.au/ai.txt