# robots.txt for Sports Pundit
# https://sportspundit.com/

User-agent: *

# Sitemaps
Sitemap: https://sportspundit.com/sitemap.xml

# Disallow admin, API, internal, and affiliate redirect routes.
# /_next/static/ (JS + CSS chunks) must stay crawlable so Googlebot can
# render pages; blocking it showed up as ~12.8K "blocked resources" in
# Site Audit and hurts rendering/CWV assessment. Everything else under
# /_next/ (data, image optimizer) stays blocked.
Disallow: /api/
Disallow: /admin/
Allow: /_next/static/
Disallow: /_next/
Disallow: /go/

# Disallow paginated/filtered pages (prevent thin content indexing)
Disallow: /*?page=*
Disallow: /*?sort=*
Disallow: /*?order=*
Disallow: /*?*sort=*&*order=*

# Disallow search results (dynamic, thin content)
Disallow: /search

# pt-BR is deployed but not launched: pages are noindex until
# NEXT_PUBLIC_LOCALE_INDEXABLE_PT_BR=true. Googlebot still spent ~25% of its
# daily crawl on them (2026-09-04..07), so block the tree outright until then.
# REMOVE this line in the same PR that flips the flag.
Disallow: /pt-br/

# Crawl delay
Crawl-delay: 1

# ── Crawlers that cost bandwidth and return nothing ─────────────────────────
# Measured in the origin log on 2026-09-18 (387K requests, 1.78 GB that day,
# against a few hundred human visits). Search engines and the AI assistants
# that cite and link to us (Google, Bing, Apple, OpenAI, Perplexity, Anthropic,
# DuckDuckGo, Exa, Linkup) are deliberately NOT listed: they stay under the
# "*" group above. SEMrush Site Audit uses its own agent (SiteAuditBot) and is
# unaffected by blocking SemrushBot, which only feeds their backlink index.
# A bot that matches a group here ignores the "*" group, hence Disallow: /.

# Meta's indexer: ~50K requests/day, the largest single declared crawler.
User-agent: meta-webindexer
User-agent: meta-externalagent
User-agent: FacebookBot
Disallow: /

# Amazon
User-agent: Amzn-SearchBot
User-agent: Amazonbot
Disallow: /

# Third-party SEO / backlink indexes
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: DataForSeoBot
User-agent: BLEXBot
User-agent: Barkrowler
User-agent: SERanking
User-agent: serpstatbot
Disallow: /

# Dataset / training-only scrapers and ad-tech
User-agent: Bytespider
User-agent: CCBot
User-agent: Reflectionbot
User-agent: ShapBot
User-agent: MathPicDatasetCrawler
User-agent: CriteoBot
User-agent: PetalBot
Disallow: /