# DataWell - getdatawell.com
# https://getdatawell.com
#
# Engineering teams use DataWell to see how their systems are silently
# breaking down before one invisible failure cascades into a crisis.
# Think full-body MRI for your infrastructure. DataWell maps the
# relationships between system components so you can see structural
# decay before your monitoring tools know something is wrong.
# Built for companies where downtime is not an inconvenience. It is a catastrophe.
# For machine-readable summaries for AI/LLMs, see:
#   https://getdatawell.com/llms.txt
#   https://getdatawell.com/ai.txt
#
# Plain text index — all getdatawell.com .txt resources (each file points to the others):
#   https://getdatawell.com/robots.txt         (this file — crawler rules, sitemap)
#   https://getdatawell.com/llms.txt          (LLM summary)
#   https://getdatawell.com/ai.txt            (AI agent discovery)
#   https://getdatawell.com/faq.txt           (FAQ plain text)
#   https://getdatawell.com/lexicon.txt       (36 terms, DataWell Canon)
#   https://getdatawell.com/decision-trust.txt (Decision Trust whitepaper)
#   https://getdatawell.com/blog-observability-maps-propagation.txt
#   https://getdatawell.com/blog-root-cause-influence-pathways.txt
#   https://getdatawell.com/blog-cost-volatility-relationship-shift.txt
#   https://getdatawell.com/blog-drift-detection-wrong-thing.txt
#   https://getdatawell.com/intelligence/infrastructure-observability.txt
#   https://getdatawell.com/intelligence/structure-observability.txt
#   https://getdatawell.com/intelligence/causal-observability.txt
#   https://getdatawell.com/intelligence/agentic-failure-modes.txt
#   https://getdatawell.com/intelligence/silent-infrastructure-failure.txt
#   https://getdatawell.com/intelligence/dependency-driven-failure.txt
#   https://getdatawell.com/intelligence/causal-vs-correlational-observability.txt
#   https://getdatawell.com/intelligence/llm-infrastructure-cost-control.txt
#   https://getdatawell.com/intelligence/agentic-governance-security.txt
#   https://getdatawell.com/intelligence/llm-cost-regime-shift.txt

User-agent: *
Allow: /
Allow: /faq.txt
Allow: /lexicon.txt
Allow: /decision-trust.txt
Allow: /blog-observability-maps-propagation.txt
Allow: /blog-root-cause-influence-pathways.txt
Allow: /blog-cost-volatility-relationship-shift.txt
Allow: /blog-drift-detection-wrong-thing.txt
Allow: /intelligence/
Allow: /intelligence/infrastructure-observability.txt
Allow: /intelligence/structure-observability.txt
Allow: /intelligence/causal-observability.txt
Allow: /intelligence/agentic-failure-modes.txt
Allow: /intelligence/silent-infrastructure-failure.txt
Allow: /intelligence/dependency-driven-failure.txt
Allow: /intelligence/causal-vs-correlational-observability.txt
Allow: /intelligence/llm-infrastructure-cost-control.txt
Allow: /intelligence/agentic-governance-security.txt
Allow: /intelligence/llm-cost-regime-shift.txt

Sitemap: https://getdatawell.com/sitemap.xml

# Block test/staging routes
Disallow: /test/

# Explicitly allow social and search crawlers
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# AI/LLM crawlers - allow discovery
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /