# robots.txt for HotelCoupons.com
# Last updated: October 2025

# Default crawl rules for all bots
User-agent: *
Allow: /

# Allow crawling of main content areas
Allow: /hotels/
Allow: /destinations/
Allow: /find-a-guide/

# Prevent crawling of duplicate content from sorting/filtering parameters
# This blocks: /hotels/city/?sortby=price but allows /hotels/city/hotel-name/
Disallow: /*?sortby=*
Disallow: /*&sortby=*
Disallow: /*?page=*
Disallow: /*&page=*

# Prevent crawling of internal search results pages
Disallow: /search?*
Disallow: /search/*
Disallow: /*?search=*
Disallow: /*?q=*

# Note: Hotel detail pages like /hotels/state/city/hotel-name/id/ are fully crawlable

# Block administrative and sensitive areas
Disallow: /admin/
Disallow: /login/
Disallow: /account/
Disallow: /checkout/
Disallow: /cart/
Disallow: /api/
Disallow: /private/

# Block common technical files
Disallow: /cgi-bin/
Disallow: /*.php$
Disallow: /*.inc$

# Block tracking and session URLs
Disallow: /*sessionid=
Disallow: /*sid=
Disallow: /*PHPSESSID=

# Block printer-friendly and mobile redirect versions if they exist
Disallow: /print/
Disallow: /*?print=
Disallow: /mobile-redirect/

# Block filters that might create duplicate content
Disallow: /*?filter=
Disallow: /*?amenities=
Disallow: /*?price=

# Special rules for Google
User-agent: Googlebot
Allow: /

# Allow Google Image indexing
User-agent: Googlebot-Image
Allow: /

# Allow Google Mobile
User-agent: Googlebot-Mobile
Allow: /

# Special rules for Bing
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Rules for other major search engines
User-agent: Slurp
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /
Crawl-delay: 2

User-agent: YandexBot
Allow: /
Crawl-delay: 2

# AI Crawlers - Allow but with considerations
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: cohere-ai
Allow: /

# Block bad bots and scrapers
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SeznamBot
Crawl-delay: 5

# Sitemap location (update with your actual sitemap URL)
Sitemap: https://www.hotelcoupons.com/sitemap.xml
Sitemap: https://www.hotelcoupons.com/sitemap-hotels.xml
Sitemap: https://www.hotelcoupons.com/sitemap-destinations.xml