# robots.txt para Webs Barcelona
# Estrategia GEO: retrieval Y training permitidos (maxima presencia en IA).
# Solo se bloquea panel/privado y scrapers agresivos sin valor (Bytespider).

User-agent: *
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /portal
Disallow: /portal/
Disallow: /api
Disallow: /api/
Disallow: /preview
Disallow: /preview/
Disallow: /pago/
Disallow: /pagar/
Disallow: /gracias
Disallow: /solicitar
Disallow: /resena
Disallow: /resenas
Disallow: /review

# [SEC] privado tambien para bots IA: cada grupo repite los Disallow del grupo *
# AI retrieval bots — permitir (traen trafico real)
User-agent: OAI-SearchBot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /portal
Disallow: /portal/
Disallow: /api
Disallow: /api/
Disallow: /preview
Disallow: /preview/
Disallow: /pago/
Disallow: /pagar/

User-agent: ChatGPT-User
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /portal
Disallow: /portal/
Disallow: /api
Disallow: /api/
Disallow: /preview
Disallow: /preview/
Disallow: /pago/
Disallow: /pagar/

User-agent: PerplexityBot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /portal
Disallow: /portal/
Disallow: /api
Disallow: /api/
Disallow: /preview
Disallow: /preview/
Disallow: /pago/
Disallow: /pagar/

User-agent: Perplexity-User
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /portal
Disallow: /portal/
Disallow: /api
Disallow: /api/
Disallow: /preview
Disallow: /preview/
Disallow: /pago/
Disallow: /pagar/

User-agent: Claude-Web
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /portal
Disallow: /portal/
Disallow: /api
Disallow: /api/
Disallow: /preview
Disallow: /preview/
Disallow: /pago/
Disallow: /pagar/

User-agent: Claude-User
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /portal
Disallow: /portal/
Disallow: /api
Disallow: /api/
Disallow: /preview
Disallow: /preview/
Disallow: /pago/
Disallow: /pagar/

User-agent: Claude-SearchBot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /portal
Disallow: /portal/
Disallow: /api
Disallow: /api/
Disallow: /preview
Disallow: /preview/
Disallow: /pago/
Disallow: /pagar/

User-agent: Google-Extended
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /portal
Disallow: /portal/
Disallow: /api
Disallow: /api/
Disallow: /preview
Disallow: /preview/
Disallow: /pago/
Disallow: /pagar/

User-agent: AppleBot-Extended
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /portal
Disallow: /portal/
Disallow: /api
Disallow: /api/
Disallow: /preview
Disallow: /preview/
Disallow: /pago/
Disallow: /pagar/

# AI training bots — PERMITIR (estrategia GEO: estar en la memoria base de los
# LLMs para ser recomendado sin necesidad de busqueda en vivo). Webs Barcelona es
# captacion/marketing: la difusion maxima beneficia, no hay contenido que proteger.
User-agent: GPTBot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /portal
Disallow: /portal/
Disallow: /api
Disallow: /api/
Disallow: /preview
Disallow: /preview/
Disallow: /pago/
Disallow: /pagar/

User-agent: ClaudeBot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /portal
Disallow: /portal/
Disallow: /api
Disallow: /api/
Disallow: /preview
Disallow: /preview/
Disallow: /pago/
Disallow: /pagar/

User-agent: CCBot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /portal
Disallow: /portal/
Disallow: /api
Disallow: /api/
Disallow: /preview
Disallow: /preview/
Disallow: /pago/
Disallow: /pagar/

User-agent: anthropic-ai
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /portal
Disallow: /portal/
Disallow: /api
Disallow: /api/
Disallow: /preview
Disallow: /preview/
Disallow: /pago/
Disallow: /pagar/

User-agent: FacebookBot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /portal
Disallow: /portal/
Disallow: /api
Disallow: /api/
Disallow: /preview
Disallow: /preview/
Disallow: /pago/
Disallow: /pagar/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /portal
Disallow: /portal/
Disallow: /api
Disallow: /api/
Disallow: /preview
Disallow: /preview/
Disallow: /pago/
Disallow: /pagar/

# Bytespider (ByteDance/TikTok) — bloqueado: agresivo y valor GEO casi nulo
User-agent: Bytespider
Disallow: /

Sitemap: https://websbarcelona.com/sitemap-index.xml

# AI / LLM resources — manifiestos para crawlers de IA generativa
# llms.txt principal:  https://websbarcelona.com/llms.txt
# llms-full.txt:       https://websbarcelona.com/llms-full.txt
# Datos estructurados: https://websbarcelona.com/datos.json