# robots.txt - Block bad bots, allow Google bots, and slow down others

# Block common bad/annoying bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: Sogou Spider
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /

User-agent: seoscanners
Disallow: /

User-agent: linkdexbot
Disallow: /

User-agent: archive.org_bot
Disallow: /

# Allow all Google bots
User-agent: Googlebot
Disallow:

User-agent: Googlebot-Image
Disallow:

User-agent: Googlebot-News
Disallow:

User-agent: Googlebot-Video
Disallow:

User-agent: AdsBot-Google
Disallow:

User-agent: AdsBot-Google-Mobile
Disallow:

# For all other bots, slow them down (crawl delay 10 seconds)
User-agent: *
Crawl-delay: 10
Disallow:

# Sitemap location
Sitemap: https://kuwaitgoldratetoday.com/sitemap_index.xml