# Exhibit — robots policy.
# Public dataset, explicitly welcome to general and AI crawlers alike.
#
# Content signals (contentsignals.org / Cloudflare convention). Exhibit is built
# to be cited, so all three uses are granted: search indexing, AI input
# (RAG / grounding / generative-search answers), and model training.
#   search=yes      build a search index and return links + excerpts
#   ai-input=yes    feed content into AI models in real time (RAG, grounding)
#   ai-train=yes    train or fine-tune AI models
# The Allow lines below are the operative permission for compliant crawlers.

User-agent: *
Content-Signal: search=yes,ai-input=yes,ai-train=yes
Allow: /

# AI/LLM crawlers — explicitly welcomed. Exhibit is built to be cited.
# (Listing these explicitly is the current convention; absence from robots.txt
# does not block them by default, but explicit Allow is the clearest signal.)
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# Sitemaps + machine-readable summary for LLMs.
Sitemap: https://exhibit509.com/sitemap.xml

# llms.txt summary (proposed standard for AI crawler signal).
# https://llmstxt.org/