# ===================================================================
# Taste of Guardians - Universal robots.txt (2025 Edition)
# ===================================================================
# Fully SEO-optimized for global search visibility and AI discovery
# Supports Google, Bing, GPTBot, Perplexity, and other major crawlers
# Allows social preview bots for enhanced link sharing

User-agent: *
Disallow: /private/
Allow: /
Allow: /index
Allow: /about-us
Allow: /offer
Allow: /onlineorder/
Allow: /gallery
Allow: /locator
Allow: /franchise
Allow: /contact
Allow: /terms-conditions
Allow: /terms
Allow: /Our-Heritage
Allow: /faq.json
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /news


# ------------------------------------------------------
# 🌍 Global Search Engines
# ------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Googlebot-News
Allow: /news

User-agent: Googlebot-Image
Allow: /images/

User-agent: Googlebot-Mobile
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

User-agent: ecosia
Allow: /

User-agent: Qwantify
Allow: /

User-agent: SeznamBot
Allow: /

User-agent: Gemini
Allow: /

User-agent: Google-Extended
Allow: /


# ------------------------------------------------------
# 🤖 AI Crawlers (LLMs, Research & Knowledge Bots)
# ------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: ia_archiver
Allow: /

User-agent: Applebot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: Pinterestbot
Allow: /

User-agent: spbot
Allow: /

User-agent: PetalBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: NeevaBot
Allow: /

# ------------------------------------------------------
# 📱 Social Media Preview Bots (OpenGraph)
# ------------------------------------------------------
User-agent: facebookexternalhit
Allow: /

User-agent: InstagramBot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Pinterestbot
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: redditbot
Allow: /

# ------------------------------------------------------
# 🧩 SEO Tools / Monitoring Bots
# ------------------------------------------------------
User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: rogerbot
Allow: /

User-agent: MJ12bot
Allow: /

User-agent: UptimeRobot
Allow: /

User-agent: Pingdom
Allow: /

User-agent: GTmetrix
Allow: /

# ------------------------------------------------------
# Sitemap & Host
# ------------------------------------------------------
Sitemap: https://tasteofguardians.com/sitemap.xml
Host: https://tasteofguardians.com