# Robots.txt for Africanews.wiki - A Wiki.News Publication
# Updated: 2025-01-14

User-agent: *
Allow: /

# Specific directives for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 2

# AI Training Bots - Allow with guidelines
User-agent: GPTBot
Allow: /
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /
Crawl-delay: 3

User-agent: anthropic-ai
Allow: /
Crawl-delay: 2

User-agent: Claude-Web
Allow: /

# News aggregators
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# Block access to admin and private areas
Disallow: /admin/
Disallow: /private/
Disallow: /.well-known/
Disallow: /api/
Disallow: /src/
Disallow: /_next/
Disallow: /node_modules/

# Allow important SEO and content files
Allow: /sitemap.xml
Allow: /rss
Allow: /favicon.png
Allow: /favicon.ico
Allow: /robots.txt
Allow: /llm.txt
Allow: /blog/*
Allow: /archives/*

# News sections - high priority crawling
Allow: /local-news
Allow: /national-news
Allow: /world-news
Allow: /politics
Allow: /business
Allow: /technology
Allow: /health
Allow: /sports
Allow: /culture

# Sitemap locations
Sitemap: https://africanews.wiki/sitemap.xml
Sitemap: https://africanews.wiki/rss

# General crawl settings
Crawl-delay: 1
Request-rate: 1/1s

# Cache directive for performance
Cache-delay: 1

# Host directive
Host: https://africanews.wiki

# Additional guidelines for AI training
# See /llm.txt for detailed AI training data usage policy