# WordPress robots.txt - Control bot crawling with delays

# Google Ads bots - allow landing pages but control crawl speed
User-agent: AdsBot-Google
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/
Disallow: /?s=
Disallow: /*?s=
Disallow: /search/
Disallow: /xmlrpc.php
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/
Disallow: /author/
Disallow: /wp-json/
Disallow: /tag/
Disallow: /category/
Crawl-delay: 60

User-agent: AdsBot-Google-Mobile
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/
Disallow: /?s=
Disallow: /*?s=
Disallow: /search/
Disallow: /xmlrpc.php
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/
Disallow: /author/
Disallow: /wp-json/
Disallow: /tag/
Disallow: /category/
Crawl-delay: 60

User-agent: AdsBot-Google-Mobile-Apps
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/
Crawl-delay: 60

# Allow Googlebot-Image but limit to essential paths
User-agent: Googlebot-Image
Allow: /wp-content/uploads/
Disallow: /
Crawl-delay: 10

# Other Google bots - more restrictive
User-agent: GoogleOther
Disallow: /
Crawl-delay: 86400

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Site-Verification
Allow: /

# Bing/Microsoft bots with controlled crawling
User-agent: bingbot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /?s=
Disallow: /*?s=
Disallow: /search/
Disallow: /xmlrpc.php
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/
Disallow: /author/
Disallow: /wp-json/
Crawl-delay: 10

User-agent: adidxbot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/
Crawl-delay: 10

# Block resource-intensive SEO bots completely
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: YandexBot
Disallow: /

# Social media crawlers - allow but control
User-agent: facebookexternalhit
Allow: /
Crawl-delay: 10

User-agent: Twitterbot
Allow: /
Crawl-delay: 10

User-agent: Amazonbot
Disallow: /
Crawl-delay: 86400

# Default rule for other bots
User-agent: *
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /?s=
Disallow: /*?s=
Disallow: /search/
Disallow: /xmlrpc.php
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/
Disallow: /author/
Disallow: /wp-json/
Crawl-delay: 60

# No sitemap needed for dynamic pages