# https://rafay.sh — agents are welcome.
# Agent guidance: /agents.md (when to use this site) and /llms.txt (index).
# Every page also serves markdown: append .md to a path, or send Accept: text/markdown.
# Feeds: /feed.xml (RSS) and /feed.json (JSON Feed). MCP server: /mcp (read-only).

User-agent: *
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

# Assistants and search indexes: allowed explicitly, so a default-deny rule
# elsewhere does not keep them off the site.
User-agent: Applebot
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

User-agent: ChatGPT-User
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

User-agent: Claude-SearchBot
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

User-agent: Claude-User
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

User-agent: ClaudeBot
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

User-agent: DeepSeekBot
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

User-agent: DuckAssistBot
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

User-agent: GPTBot
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

User-agent: Meta-ExternalFetcher
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

User-agent: MistralAI-User
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

User-agent: OAI-SearchBot
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

User-agent: Perplexity-User
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

User-agent: PerplexityBot
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

User-agent: ora-agent
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

# Bulk collectors: no crawl, per the ai-train=no signal above.
User-agent: Amazonbot
Content-Signal: search=yes,ai-train=no,use=reference
Disallow: /

User-agent: Applebot-Extended
Content-Signal: search=yes,ai-train=no,use=reference
Disallow: /

User-agent: Bytespider
Content-Signal: search=yes,ai-train=no,use=reference
Disallow: /

User-agent: CCBot
Content-Signal: search=yes,ai-train=no,use=reference
Disallow: /

User-agent: FacebookBot
Content-Signal: search=yes,ai-train=no,use=reference
Disallow: /

User-agent: Google-CloudVertexBot
Content-Signal: search=yes,ai-train=no,use=reference
Disallow: /

User-agent: Google-Extended
Content-Signal: search=yes,ai-train=no,use=reference
Disallow: /

User-agent: PetalBot
Content-Signal: search=yes,ai-train=no,use=reference
Disallow: /

User-agent: meta-externalagent
Content-Signal: search=yes,ai-train=no,use=reference
Disallow: /

Sitemap: https://rafay.sh/sitemap.xml