# AnnuaireCI — robots.txt
# https://annuaireci.com
#
# NOTE: this file lives in the web root on the server
# (/sites/annuaireci.com/files/robots.txt). deploy-to-spinupwp.sh only syncs
# mu-plugins/ and themes/, so this repo copy is for version control — update the
# server copy too. A physical file here overrides WordPress's virtual robots.txt.
#
# Cloudflare prepends its own AI-crawler block ("Cloudflare Managed Content")
# to whatever this file returns; that section is managed in the Cloudflare
# dashboard under AI Crawl Control, not here.

User-agent: *
Allow: /

# Admin and internals
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/mu-plugins/
Disallow: /wp-content/cache/
Disallow: /wp-json/

# Assets needed to render pages correctly, and business images
Allow: /wp-content/themes/
Allow: /wp-content/uploads/
Allow: /wp-includes/js/
Allow: /wp-includes/css/

# Internal search results (duplicate content)
Disallow: /?s=
Disallow: /*?s=
Disallow: /search/

# Feeds, trackbacks, author archives
Disallow: /feed/
Disallow: /*/feed/
Disallow: /comments/feed/
Disallow: /trackback/
Disallow: /author/

# Faceted / query-string variants of directory listings
Disallow: /categorie/*?
Disallow: /ville/*?

# Business claim form. Every listing links to it with its own business_id, so a
# full crawl of 40k listings becomes 40k uncached hits on the origin (195k in
# five days, Sep 2026). There is nothing to index there.
Disallow: /revendiquer-entreprise/
Disallow: /en/revendiquer-entreprise/
Disallow: /es/revendiquer-entreprise/
Disallow: /*/revendiquer-entreprise/

Sitemap: https://annuaireci.com/sitemap.xml

# ─────────────────────────────────────────────────────────────────────────────
# Changes from the pre-migration robots.txt, and why
#
# 1. REMOVED the per-bot groups:
#        User-agent: Googlebot      / Allow: /
#        User-agent: Bingbot        / Allow: /
#        User-agent: facebookexternalhit, Twitterbot, LinkedInBot ...
#    A crawler obeys ONLY the most specific User-agent group that matches it.
#    Those groups contained nothing but "Allow: /", so Googlebot and Bingbot
#    ignored every Disallow above — wp-admin, search results, feeds, the lot.
#    The intent was "make sure Google can crawl"; the effect was "Google ignores
#    all our rules". Deleting them makes the * group apply to everyone.
#
# 2. REMOVED "Disallow: /page/4..10/" and "Disallow: /page/*/".
#    The comment said "block pagination beyond page 3", but the wildcard blocked
#    ALL pagination. On a directory with 41,000+ businesses that stops crawlers
#    reaching almost every listing. robots.txt cannot express "beyond page 3";
#    crawl budget is better handled with rel=next/prev and the sitemap.
#
# 3. FIXED the sitemap lines. Previously advertised:
#        https://annuaireci.com/wp-sitemap.xml     (redirects)
#        https://annuaireci.com/sitemap_index.xml  (redirects)
#    and WordPress's virtual file advertised /sitemap.rss, which 404s.
#    Only /sitemap.xml is canonical and returns 200.
#
# 4. ADDED "Allow: /wp-admin/admin-ajax.php" (front-end AJAX) and
#    "Allow: /wp-content/uploads/" so business images can be indexed.
# ─────────────────────────────────────────────────────────────────────────────