# robots.txt — Futuro das Carreiras
# Portal editorial brasileiro de carreira na era da IA.
# Estratégia: liberar TUDO para bots de IA, restringir endpoints técnicos
# apenas para crawlers de busca tradicional (que não precisam consumir API).

# ─────────────────────────────────────────────────────────────
# Bots de IA / LLMs — acesso TOTAL liberado
# (eles precisam de /api/, /mcp/, /llms-full.txt, /dataset.json)
# ─────────────────────────────────────────────────────────────

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Cohere-Crawler
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bytespider
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Omgilibot
Allow: /

User-agent: Timpibot
Allow: /

User-agent: YouBot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: MistralAI-User
Allow: /

# ─────────────────────────────────────────────────────────────
# Search engines tradicionais — restringir endpoints técnicos
# (a indexação dos JSON/MCP polui o índice sem benefício)
# ─────────────────────────────────────────────────────────────

User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /mcp/
Disallow: /llms-full.txt
Disallow: /dataset.json

User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /mcp/
Disallow: /llms-full.txt
Disallow: /dataset.json

User-agent: DuckDuckBot
Allow: /
Disallow: /api/
Disallow: /mcp/

User-agent: YandexBot
Allow: /
Disallow: /api/
Disallow: /mcp/

# ─────────────────────────────────────────────────────────────
# Default — todos os outros bots
# ─────────────────────────────────────────────────────────────

User-agent: *
Allow: /
Disallow: /api/
Disallow: /mcp/
Disallow: /llms-full.txt
Disallow: /dataset.json
Disallow: /.well-known/

# ─────────────────────────────────────────────────────────────
# Sitemaps e endpoints IA-first
# ─────────────────────────────────────────────────────────────

Sitemap: https://futurodascarreiras.com/sitemap.xml

# Endpoints IA-first (referência para descoberta):
# - https://futurodascarreiras.com/llms.txt
# - https://futurodascarreiras.com/llms-full.txt
# - https://futurodascarreiras.com/dataset.json
# - https://futurodascarreiras.com/api/openapi.json
# - https://futurodascarreiras.com/mcp/
# - https://futurodascarreiras.com/.well-known/mcp.json
# - https://futurodascarreiras.com/.well-known/ai-plugin.json