# Wealth Architect - Robots Configuration
# https://wealth-architect.eu

# ─── Traditional Search Crawlers ───────────────────────────────
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# ─── Tier 1: Critical AI Search Crawlers (ALLOW) ──────────────
# These power the AI search products where users actively look for answers.

User-agent: GPTBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /dashboard
Disallow: /reset-password
Disallow: /results

User-agent: OAI-SearchBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /dashboard
Disallow: /reset-password
Disallow: /results

User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /dashboard
Disallow: /reset-password
Disallow: /results

User-agent: ClaudeBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /dashboard
Disallow: /reset-password
Disallow: /results

User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /dashboard
Disallow: /reset-password
Disallow: /results

# ─── Tier 2: Broader AI Ecosystem (ALLOW) ─────────────────────
# These serve large AI platforms or search ecosystems.

User-agent: Google-Extended
Allow: /
Disallow: /admin
Disallow: /dashboard

User-agent: GoogleOther
Allow: /
Disallow: /admin
Disallow: /dashboard

User-agent: Applebot-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /dashboard

User-agent: Amazonbot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /dashboard

User-agent: FacebookBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /dashboard

# ─── Tier 3: Training-Only Crawlers ───────────────────────────
# CCBot and anthropic-ai are training-only; allowed for long-term presence.
# Bytespider is blocked (aggressive crawling, low visibility benefit).

User-agent: Anthropic-AI
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /dashboard

User-agent: CCBot
Allow: /
Disallow: /admin
Disallow: /dashboard

User-agent: Bytespider
Disallow: /

User-agent: cohere-ai
Allow: /
Disallow: /admin
Disallow: /dashboard

# ─── Default (All Other Crawlers) ─────────────────────────────
User-agent: *
Allow: /
Allow: /diagnostic
Allow: /financial-planning
Allow: /glossary
Allow: /auth
Allow: /for/
Disallow: /admin
Disallow: /dashboard
Disallow: /reset-password
Disallow: /results
Disallow: /report/
Disallow: /gift/
Disallow: /*?v=
Disallow: /*?seg=

Sitemap: https://wealth-architect.eu/sitemap.xml