# ============================================================
# robots.txt — https://aikb.scighera.com (AIKB — Anthilla Knowledge Base)
# AI Crawler Management with differential Crawl-delay
# Updated: 2026-10-05 (S45-40)
# Agent discovery: /onboard /agent-manifest.json /openapi.json /llms.txt
# ============================================================

User-agent: *
Allow: /
Disallow: /api/
Disallow: /mcp
Disallow: /cli/
Disallow: /dist/
Crawl-delay: 1

# === SEARCH CRAWLERS ===
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /mcp

User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /mcp

# === AI CRAWLERS — WELCOME ===
User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /mcp
Crawl-delay: 10

User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /mcp
Crawl-delay: 10

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /mcp
Crawl-delay: 15

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /mcp
Crawl-delay: 10

User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /mcp
Crawl-delay: 15

User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /mcp

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /mcp
Crawl-delay: 10

User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /mcp
Crawl-delay: 10

User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /mcp
Crawl-delay: 10

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api/
Disallow: /mcp
Crawl-delay: 10

User-agent: MistralAI-User
Allow: /
Disallow: /api/
Disallow: /mcp
Crawl-delay: 15

User-agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /mcp
Crawl-delay: 10

User-agent: DuckAssistBot
Allow: /
Disallow: /api/
Disallow: /mcp
Crawl-delay: 10

Sitemap: https://aikb.scighera.com/sitemap.xml
Sitemap: https://aikb.scighera.com/sitemap-ai.xml