# robots.txt for serppro.ai
# Sitemap: https://www.serppro.ai/sitemap.xml
# LLM directory: https://www.serppro.ai/llms.txt

User-agent: *
# Content usage preferences (contentsignals.org): allow search indexing,
# use in AI answers/RAG, and model training. We want the citations and we
# want the brand present in model knowledge, so all three are yes.
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Block dev/build artifacts and utility pages
Disallow: /draft/
Disallow: /assets/
Disallow: /cdn-cgi/
Disallow: /i18n/
Disallow: /404
Disallow: /deposit-bonus-embed
Disallow: /deposit-bonus-table
Disallow: /dark-preview
Disallow: /serppro-landing-v2
Disallow: /_backup_path2/

# Major search bots — explicit allow
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

# AI crawlers — allow access (you can opt out by reversing these)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# powers ChatGPT's search citations, distinct from GPTBot
User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: CCBot
Allow: /

# Bad/scraper bots — block
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SeznamBot
Disallow: /

Sitemap: https://www.serppro.ai/sitemap.xml