# ============================================================
# robots.txt — https://flable.ai
# Last updated: 2026-09-15
# ============================================================

# Default policy — allow everything
User-agent: *
Allow: /

# ────────────────────────────────────────────────────────────
# AI TRAINING CRAWLERS
# These bots scrape content for model training datasets.
# Allowing them increases the chance your content appears in
# AI model knowledge and is cited in AI-generated answers.
# ────────────────────────────────────────────────────────────
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: cohere-ai
Allow: /

# ────────────────────────────────────────────────────────────
# AI RETRIEVAL / SEARCH CRAWLERS
# These bots fetch content in real time to ground AI answers
# with citations. Blocking them = never getting cited in
# ChatGPT search, Perplexity answers, or Claude search.
# ────────────────────────────────────────────────────────────
User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot-Extended
Allow: /

# ────────────────────────────────────────────────────────────
# AI USER-TRIGGERED FETCHERS
# These fire when a user explicitly asks an AI about a URL.
# Blocking them means even direct questions about your site
# will return "I can't access that page."
# ────────────────────────────────────────────────────────────
User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Perplexity-User
Allow: /

# ────────────────────────────────────────────────────────────
# STANDARD SEARCH ENGINE CRAWLERS
# ────────────────────────────────────────────────────────────
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# ────────────────────────────────────────────────────────────
# BLOCK KNOWN BAD BOTS
# ────────────────────────────────────────────────────────────
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# Host
Host: https://flable.ai

# Sitemaps
Sitemap: https://flable.ai/sitemap.xml