User-agent: *
Allow: /tools/
Allow: /
Disallow: /api/
Sitemap: https://stanzaapi.com/sitemap.xml
Sitemap: https://stanzaapi.com/tools/sitemap.xml

# Search Engine Crawlers
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

# Generative AI, Search & LLM Web Crawlers
User-agent: Google-Extended
Allow: /
User-agent: GPTBot
Allow: /
User-agent: OAI-SearchBot
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: Applebot-Extended
Allow: /
User-agent: Amazonbot
Allow: /

# LLMs Text Index (llmstxt.org)
# Location: https://stanzaapi.com/tools/llms.txt
# Full Reference: https://stanzaapi.com/tools/llms-full.txt