# Evomon Wiki — robots.txt
# Traditional search engines (Google, Bing, DuckDuckGo, …) are welcome on all
# public content; /api, /auth and /admin are always closed. AI answer/retrieval
# bots may read public pages, while AI training crawlers remain blocked.
# NOTE: this does NOT affect Google Search ranking — Googlebot is untouched;
# only Google's SEPARATE AI-training bot (Google-Extended) is blocked.

User-agent: *
Disallow: /api/
Disallow: /auth/
Disallow: /admin/
Allow: /
# Content Signals (https://contentsignals.org/): welcome traditional search
# indexing and answer retrieval; decline AI model training.
Content-Signal: search=yes, ai-train=no, ai-input=yes

# --- AI answer / search / retrieval engines — public pages allowed ---
User-agent: OAI-SearchBot
Disallow: /api/
Disallow: /auth/
Disallow: /admin/
Allow: /

User-agent: ChatGPT-User
Disallow: /api/
Disallow: /auth/
Disallow: /admin/
Allow: /

User-agent: PerplexityBot
Disallow: /api/
Disallow: /auth/
Disallow: /admin/
Allow: /

User-agent: Claude-SearchBot
Disallow: /api/
Disallow: /auth/
Disallow: /admin/
Allow: /

User-agent: Claude-User
Disallow: /api/
Disallow: /auth/
Disallow: /admin/
Allow: /

# --- AI training crawlers — DISALLOWED ---
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

# --- Aggressive / low-value scrapers (heavy crawl load, no traffic back) — DISALLOWED ---
# NOTE: facebookexternalhit (Meta's LINK-PREVIEW bot that renders evomon.wiki
# previews in Discord / Messenger / WhatsApp) is deliberately NOT listed, so it
# stays allowed via `User-agent: *`. Only Meta's AI-training agent is blocked.
User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# Applebot is Apple's traditional SEARCH crawler (Siri / Spotlight) — the Apple
# equivalent of Googlebot, NOT an AI scraper — so it stays allowed to preserve
# Apple search discovery. Applebot-Extended is explicitly blocked above.
User-agent: Applebot
Disallow: /api/
Disallow: /auth/
Disallow: /admin/
Allow: /

Sitemap: https://www.evomon.wiki/sitemap.xml