# =============================================
# robots.txt - mateco.es
# Actualizado: mayo 2025
# =============================================

# ------------------------------------------
# CRAWLERS GENERALES
# ------------------------------------------
User-agent: *

# WordPress - zonas tecnicas y de administracion
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# WooCommerce - archivos internos sin valor SEO
Disallow: /wp-content/uploads/wc-logs/
Disallow: /wp-content/uploads/woocommerce_transient_files/
Disallow: /wp-content/uploads/woocommerce_uploads/

# Uploads accesibles (imagenes de producto, catalogos)
Allow: /wp-content/uploads/
Allow: /wp-includes/js/

# Tags de WordPress (sin valor comercial)
Disallow: /tag/

# Archivos internos de WordPress
Disallow: /feed/
Disallow: /?feed=
Disallow: /?s=

# Paginas de autor (marcadas con noindex en el sitio)
Disallow: /author/

# Formulario de empleo con parametros (error 500)
Disallow: /?feed=job_feed

# Noticias - seccion permitida, filtros bloqueados
Allow: /noticias/
Disallow: /noticias/?*date=
Disallow: /noticias/?*tax_post_tag=
Disallow: /noticias/?*tax_category=
Disallow: /noticias/?*unfilter=
Disallow: /noticias/*?*date=
Disallow: /noticias/*?*tax_post_tag=
Disallow: /noticias/*?*tax_category=
Disallow: /noticias/*?*unfilter=

# Blog - seccion permitida, filtros bloqueados
Allow: /blog/
Disallow: /blog/?*cat=
Disallow: /blog/?*tag=
Disallow: /blog/*?*cat=
Disallow: /blog/*?*tag=


# ------------------------------------------
# AI CRAWLERS - estrategia GEO activa
# Acceso COMPLETO al catalogo de mateco.
# Objetivo: visibilidad en ChatGPT, Gemini,
# Perplexity y resto de IAs generativas.
# ------------------------------------------

# --- OPENAI ---
# GPTBot: entrenamiento de modelos
User-agent: GPTBot
Allow: /

# OAI-SearchBot: busqueda y citas en ChatGPT
User-agent: OAI-SearchBot
Allow: /

# ChatGPT-User: navegacion en tiempo real
User-agent: ChatGPT-User
Allow: /

# --- ANTHROPIC (Claude) ---
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# --- GOOGLE ---
# Google-Extended: Gemini / Vertex AI / AI Overviews
User-agent: Google-Extended
Allow: /

# --- PERPLEXITY ---
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# --- META (Llama / Meta AI) ---
User-agent: Meta-ExternalAgent
Allow: /

User-agent: FacebookBot
Allow: /

# --- APPLE (Siri / Apple Intelligence) ---
User-agent: Applebot-Extended
Allow: /

# --- COHERE ---
User-agent: cohere-ai
Allow: /

User-agent: cohere-training-data-crawler
Allow: /

# --- YOU.COM ---
User-agent: YouBot
Allow: /

# --- DUCKDUCKGO ---
User-agent: DuckAssistBot
Allow: /

# --- COMMON CRAWL ---
User-agent: CCBot
Allow: /

# --- BYTEDANCE / TIKTOK ---
User-agent: Bytespider
Allow: /


# ------------------------------------------
# SITEMAP
# ------------------------------------------
Sitemap: https://www.mateco.es/sitemap_index.xml