# Robots.txt Generated automatically for indexnow.fr
# Allow all crawlers to access all content
User-agent: *
Allow: /

# ===== CRAWL RATE CONTROL =====
# Limit aggressive SEO bots to prevent server overload

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: BLEXBot
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 15

# ===== BLOCK HARMFUL/SPAM BOTS =====
# These bots consume bandwidth without providing value

User-agent: SemrushBot-SA
Disallow: /

User-agent: SemrushBot-BA
Disallow: /

User-agent: SemrushBot-SI
Disallow: /

User-agent: SemrushBot-SWA
Disallow: /

User-agent: SemrushBot-CT
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

# Block aggressive scrapers
User-agent: SeznamBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DotBot
Disallow: /

# Block social media bots that don't respect rate limits
User-agent: facebookexternalhit
Crawl-delay: 5

# ===== SEARCH ENGINES - WELCOME =====
# Explicitly welcome major search engines

User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: YandexBot
Allow: /
Crawl-delay: 2

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 2

# ===== SITEMAPS =====
Sitemap: https://indexnow.fr/sitemap.xml