User-agent: *
Allow: /

# Sitemap location
Sitemap: https://asktodo.ai/sitemap.xml

# Crawl-delay for respectful crawling
Crawl-delay: 1

# Block sensitive areas
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /static/
Disallow: /image genration/

# Explicitly allow important pages and content
Allow: /
Allow: /tools/
Allow: /blog/
Allow: /blog/*
Allow: /pricing
Allow: /contact
Allow: /privacy
Allow: /terms
Allow: /ads.txt
Allow: /sitemap.xml
Allow: /manifest.json

# Block image generation folder (duplicate removed)
Disallow: /image genration/

# Search engine specific optimizations
User-agent: Googlebot
Allow: /
Allow: /blog/
Allow: /blog/*
Allow: /tools/
Crawl-delay: 1

User-agent: Bingbot  
Allow: /
Allow: /blog/
Allow: /blog/*
Allow: /tools/
Crawl-delay: 1

# Social media crawlers for better sharing
User-agent: facebookexternalhit
Allow: /
Allow: /blog/
Allow: /tools/

User-agent: Twitterbot
Allow: /
Allow: /blog/
Allow: /tools/

User-agent: LinkedInBot
Allow: /
Allow: /blog/
Allow: /tools/

# AdSense crawler
User-agent: Mediapartners-Google
Allow: /
Allow: /blog/
Allow: /tools/