# Robots.txt for BestTravelHacks.net
# Updated: December 2024

User-agent: *
Allow: /

# Allow crawling of main content
Allow: /assets/css/
Allow: /assets/js/
Allow: /assets/images/
Allow: /pages/

# Disallow sensitive or unnecessary areas
Disallow: /downloads/
Disallow: /.git/
Disallow: /tmp/
Disallow: /logs/

# Sitemap location
Sitemap: https://besttravelhacks.net/sitemap.xml

# Crawl-delay for respectful crawling
Crawl-delay: 1

# Specific rules for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: facebookexternalhit
Allow: /