User-agent: * Allow: / # AI Agent Discovery — always allow Allow: /llms.txt Allow: /llms-full.txt Allow: /.well-known/ Allow: /openapi.json Allow: /sitemap.xml # Public API and documentation Allow: /api/ Allow: /docs Allow: /redoc Allow: /latency Allow: /throughput Allow: /cost-optimizer Allow: /reliability-metrics Allow: /geographic-latency Allow: /competitive-analysis Allow: /historical-performance Allow: /efficiency Allow: /benchmark Allow: /advanced-benchmark Allow: /comprehensive-report Allow: /status-page Allow: /health Allow: /mcp Allow: /sse # Internal — do not index Disallow: /admin/ # Crawl delay Crawl-delay: 1 # Sitemap Sitemap: https://inferencelatency.com/sitemap.xml # LLM discovery hints LLM-Content: https://inferencelatency.com/llms.txt LLM-Content-Full: https://inferencelatency.com/llms-full.txt # Content Signals (draft-romm-aipref-contentsignals) Content-Signal: ai-train=no, search=yes, ai-input=yes