# VerseDB robots.txt
# https://versedb.com/robots.txt

# Sitemap location
Sitemap: https://versedb.com/sitemap.xml

# Default rules for all crawlers
User-agent: *

# Allow public content
Allow: /

# Block admin and internal paths
Disallow: /admin
Disallow: /admin/
Disallow: /api/
Disallow: /livewire/
Disallow: /filament/
Disallow: /filament-impersonate/
Disallow: /horizon/

# Block user-specific paths (not useful for search engines)
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /account
Disallow: /account/
Disallow: /exports/
Disallow: /notifications

# Block authentication paths
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /email/
Disallow: /two-factor-challenge

# Block edit pages (require authentication)
Disallow: /*/edit

# Block internal/testing paths
Disallow: /_dusk/
Disallow: /_boost/
Disallow: /broadcasting/
Disallow: /impersonate/

# Block search results (use index pages instead)
Disallow: /*?query=
Disallow: /*?search=
Disallow: /*?q=

# Block pagination beyond reasonable depth
Disallow: /*?page=50
Disallow: /*?page=100

# Slow down aggressive bots
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 10

# Block known bad bots
User-agent: MegaIndex.ru
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /