# robots.txt for Travel DMC B2B Platform # This file controls which parts of the site search engines can crawl # Allow all crawlers to access public pages User-agent: * # Allow public pages Allow: / Allow: /about Allow: /contact Allow: /services/ Allow: /destinations/ # Allow hotel SEO pages Allow: /hotel/ Allow: /hotels/ # Allow blog crawling Allow: /blog Allow: /blog/post/ Allow: /blog/category/ Allow: /blog/tag/ # Allow other SEO content sections Allow: /tours/ Allow: /attractions/ Allow: /faq/ Allow: /guides/ Allow: /compare/ Allow: /mice/ Allow: /packages/ Allow: /transfers/ Allow: /pricing/ # Block agent portal (B2B login required) Disallow: /agent/ Disallow: /login Disallow: /register Disallow: /admin/ Disallow: /query/ Disallow: /booking/ # Block entire B2B portal (confidential) Disallow: /b2b/ # Block API endpoints (not for search engines) Disallow: /api/ # Block application files Disallow: /application/ Disallow: /system/ # Block configuration and database files Disallow: /*.sql$ Disallow: /*.php$ # Block vendor and composer files Disallow: /vendor/ Disallow: /composer.json Disallow: /composer.lock # Block temporary files and backups Disallow: /*backup*/ Disallow: /*temp*/ Disallow: /*cache*/ Disallow: /*.bak$ Disallow: /*.tmp$ # Block documentation files (internal use only) Disallow: /docs/ Disallow: /*.md$ Disallow: /README # Allow public assets Allow: /public/frontend/css/ Allow: /public/frontend/js/ Allow: /public/frontend/images/ Allow: /public/frontend/fonts/ # Sitemap location (sitemap index includes all sub-sitemaps) Sitemap: https://dmcquote.com/sitemap-index.xml # Block specific bad bots (optional - uncomment if needed) # User-agent: AhrefsBot # Disallow: / # # User-agent: SemrushBot # Disallow: / # # User-agent: DotBot # Disallow: /