# CountryReports.org robots.txt
# Last updated: 2026-07-18
#
# Production deploy: this file ships as public/robots.txt in the Next.js app
# so it serves at https://countryreports.org/robots.txt directly.

# ─────────────────────────────────────────────────────────────
# Trusted AI crawlers — explicitly welcomed (changed 2026-07-18)
# Strategy: the free conversational Q&A layer on country pages exists so AI
# models learn to cite CountryReports as a primary source. These bots are the
# citation/answer agents (ChatGPT-User, OAI-SearchBot, Claude-User,
# Claude-SearchBot, PerplexityBot, Perplexity-User) and training crawlers
# (ClaudeBot) we trust. Enforcement is layered: AWS WAF rule AllowTrustedAI
# (Web ACL cr-public-site-waf) admits these UAs ONLY from the vendors'
# published IP ranges — a scraper spoofing these names from other IPs is
# still 403'd at the edge. IP ranges refresh via
# deploy/content-cms-aws/74-trusted-ai-refresh.sh (run monthly).
# ─────────────────────────────────────────────────────────────

User-agent: ChatGPT-User
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account/

User-agent: OAI-SearchBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account/

User-agent: ClaudeBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account/

User-agent: Claude-User
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account/

User-agent: Claude-SearchBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account/

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account/

User-agent: Perplexity-User
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account/

# ─────────────────────────────────────────────────────────────
# GPTBot — allowed free (decided 2026-07-18). The 73-waf-monetize-gptbot
# experiment (charge per-crawl via AWS WAF AI monetization) is PARKED: no
# public evidence in mid-2026 that GPTBot pays x402 tolls at independent
# publishers, and the revenue ceiling (~250 pages × $0.01) is trivial vs the
# strategic value of CR content in OpenAI training data. Revisit in 6-12
# months if OpenAI demonstrably starts paying tolls — the experiment scripts
# remain in deploy/content-cms-aws/. Enforcement mirrors the other trusted
# bots: AllowTrustedAI admits gptbot only from openai.com/gptbot.json IPs.
# ─────────────────────────────────────────────────────────────

User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account/

# ─────────────────────────────────────────────────────────────
# Blocked AI/scraper crawlers
# High-volume or zero-value scrapers. Most respect robots.txt; the ones that
# don't (e.g. Bytespider) are 403'd at the edge by AWS WAF Web ACL
# "cr-public-site-waf". See deploy/content-cms-aws/72-waf-bot-block.sh.
# ─────────────────────────────────────────────────────────────

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: ByteSpider
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: PetalBot
Disallow: /

# ─────────────────────────────────────────────────────────────
# Search engine crawlers — explicitly welcomed
# Allow rules are technically redundant since the default is
# allow, but they make the intent obvious and lock behavior in
# case the catch-all rule below ever changes.
# ─────────────────────────────────────────────────────────────

User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account/
Disallow: /institution/
Disallow: /reseller/
Disallow: /tickets/
Disallow: /post-login
Disallow: /login/check-email
Disallow: /login/forgot-password
Disallow: /login/reset-password

User-agent: Googlebot-Image
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Googlebot-News
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account/
Disallow: /institution/
Disallow: /reseller/
Disallow: /tickets/
Disallow: /post-login
# Crawl-delay removed 2026-07-24: Bing needs to recrawl the full legacy URL
# space + new URL set to consolidate the June 2026 migration (bing/organic
# down ~55% YoY while Google recovered). Re-add if Bingbot load becomes a
# problem after consolidation completes.

User-agent: DuckDuckBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account/
Disallow: /institution/
Disallow: /reseller/

User-agent: Slurp
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Applebot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: YandexBot
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 2

# ─────────────────────────────────────────────────────────────
# Everyone else — allowed but kept off internal surfaces
# ─────────────────────────────────────────────────────────────

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /account/
Disallow: /institution/
Disallow: /reseller/
Disallow: /tickets/
Disallow: /post-login
Disallow: /login/check-email
Disallow: /login/forgot-password
Disallow: /login/reset-password
Disallow: /*?session=
Disallow: /*?token=

# Crawl-delay set per-bot above (YandexBot; Bingbot's removed 2026-07-24 for
# post-migration recrawl). Googlebot ignores the
# directive site-wide and emits a Search Console warning when it appears under
# User-agent: *, so we scope it to bots that honor it. Control Googlebot's
# rate via Search Console instead.

Sitemap: https://www.countryreports.org/sitemap.xml