# robots.txt for https://usewrit.app
# Policy: Writ is an API & MCP layer for AI agents. We WANT to be discoverable
# and citable by AI answer engines, so reputable AI crawlers are allowed by
# default. The app (app.usewrit.app) is excluded at its own origin.
# Curated AI map: https://usewrit.app/llms.txt

# ---------------------------------------------------------------------------
# Default: all crawlers may access public marketing / docs / marketplace pages.
# ---------------------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /draft/
Disallow: /preview/
Disallow: /internal/
# Development surfaces: component/mockup harnesses, not product pages. They also
# carry <meta robots="noindex">; this keeps them out of the crawl budget too.
Disallow: /dev/
Disallow: /fr/dev/
Disallow: /es/dev/
Disallow: /*?*sort=
Disallow: /*?*filter=
Disallow: /search

# ---------------------------------------------------------------------------
# Search engines (explicit allow; manage crawl rate in Search/Bing consoles).
# ---------------------------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

# ---------------------------------------------------------------------------
# AI training / answer-engine crawlers — ALLOWED (GEO strategy: we want
# citations in ChatGPT, Claude, Perplexity, Google AI Overviews, etc.).
# ---------------------------------------------------------------------------
# OpenAI
User-agent: GPTBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: OAI-SearchBot
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /
User-agent: anthropic-ai
Allow: /
User-agent: Claude-Web
Allow: /
User-agent: Claude-User
Allow: /
User-agent: Claude-SearchBot
Allow: /

# Google (Gemini / AI Overviews opt-in signal)
User-agent: Google-Extended
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity-User
Allow: /

# Common Crawl (feeds many LLM training sets)
User-agent: CCBot
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /

# Amazon
User-agent: Amazonbot
Allow: /

# Meta
User-agent: Meta-ExternalAgent
Allow: /
User-agent: meta-externalagent
Allow: /
User-agent: FacebookBot
Allow: /

# ---------------------------------------------------------------------------
# Reasoned exception — Bytespider (ByteDance). Allowed by default for citation
# reach. Uncomment the Disallow if logs show abusive crawl-rate or no value.
# ---------------------------------------------------------------------------
User-agent: Bytespider
Allow: /
# Disallow: /

# ---------------------------------------------------------------------------
# Sitemaps.
# ---------------------------------------------------------------------------
Sitemap: https://usewrit.app/sitemap-index.xml

# Legacy / Yandex only (ignored by Google):
Host: usewrit.app