# Asupan Dood - robots.txt
# SEO Configuration for search engine crawlers

User-agent: *
Allow: /
Allow: /categories
Allow: /tags
Allow: /video/
Allow: /sitemap.xml
Disallow: /api/
Disallow: /admin/
Disallow: /_astro/
Disallow: /.env
Disallow: /package.json

# Sitemap reference
Sitemap: https://asupandood.media/sitemap.xml

# Google crawler - faster crawling allowed
User-agent: Googlebot
Allow: /
Crawl-delay: 0

# Bing crawler - moderate crawl delay
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Other crawlers
User-agent: *
Crawl-delay: 2
Request-rate: 1/5