User-agent: *
Allow: /
Disallow: /api/
Disallow: /studio-deposit-harness*

# /dashboard, /settings and /my-agents are signed-in surfaces and every one of
# their pages carries <meta name="robots" content="noindex">. A Disallow here
# would hide that tag from the crawler that needs to read it: Google kept them
# in the index as bare URLs ("Indexed, though blocked by robots.txt") precisely
# because it was never allowed to fetch the page and see the noindex. Crawling
# is what removes them, so leave these open.

# Embed routes serve raw runtime, not human content; keep them out of SERPs.
Disallow: /agent/*/embed
Disallow: /widget/*/embed
Disallow: /avatar/*/embed

# Cosmetic OG image endpoints: fine for social crawlers, useless for SERP.
Disallow: /api/agent/*/og
Disallow: /api/avatar/*/og

# ----------------------------------------------------------------------
# AI assistants fetching on behalf of a live user (ChatGPT, Claude, Grok,
# Perplexity, Kimi, DuckDuckGo, Mistral) are welcome everywhere public,
# INCLUDING the API: the x402 pay-per-call endpoints, the free 3D lanes,
# and the MCP servers are the product. Only signed-in account surfaces
# stay off limits.
# ----------------------------------------------------------------------
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Perplexity-User
User-agent: DuckAssistBot
User-agent: MistralAI-User
Allow: /
Disallow: /my-agents
Disallow: /dashboard
Disallow: /settings
Disallow: /studio-deposit-harness*

# AI index and training crawlers (GPTBot, ClaudeBot, GrokBot, PerplexityBot,
# Google-Extended, Bytespider, CCBot, and the rest) are equally welcome and
# already covered by the "User-agent: *" rules above; nothing here blocks
# them. Start from the machine-readable index instead of scraping HTML:
#
#   /llms.txt                      concise AI index (https://llmstxt.org/)
#   /llms-full.txt                 long-form page index
#   /.well-known/mcp.json          hosted MCP servers
#   /.well-known/agent-card.json   A2A agent card
#   /.well-known/x402.json         paid-service catalog (x402)
#   /.well-known/openapi.yaml      HTTP API spec

Sitemap: /sitemap.xml