# GENOSYS - Professional Korean Dermacosmetics
# Official UAE Distributor (Genosys Middle East FZ-LLC) - Microneedling Devices & Skincare Products
# AI-friendly: llms.txt available at https://genosys.ae/llms.txt
# Last updated: 2026-05-11

# SITEMAPS - Place at top for crawler discovery
Sitemap: https://genosys.ae/sitemap.xml

# ============================================================
# DEFAULT CRAWL RULES
# ============================================================

User-agent: *
Allow: /

# Allow multilingual content explicitly
Allow: /ar/
Allow: /ru/

# Allow important content directories
Allow: /products/
Allow: /blog/
Allow: /locations/
Allow: /documents/
Allow: /images/
Allow: /videos/
Allow: /faq/
Allow: /about
Allow: /brand
Allow: /training
Allow: /contact
Allow: /delivery
Allow: /genosys
Allow: /partners
Allow: /skin-recommendation
Allow: /guides/

# Allow concern-based and category landing pages
Allow: /products/concern/
Allow: /products/category/

# Allow optimized SEO pages
Allow: /korean-dermacosmetics-products/
Allow: /about-genosys-middle-east
Allow: /genosys-brand-story
Allow: /professional-skincare-training
Allow: /contact-genosys-uae
Allow: /delivery-shipping-uae
Allow: /genosys-official
Allow: /professional-documents

# Allow essential files
Allow: /favicon.ico
Allow: /manifest.json
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /ai-products.txt
Allow: /opensearch.xml
Allow: /feed/products.xml
Allow: /feed/blog.xml
Allow: /feed/blog.atom

# Disallow admin and private areas.
# Each localized path is listed twice: robots.txt matches on prefix, so `/cart/`
# does not cover `/ar/cart/` or `/ru/cart/`. The `/*/cart/` form catches those and
# any locale added later. Pages we want dropped from the index are deliberately NOT
# listed here — a crawler blocked from fetching them can never see their noindex.
Disallow: /admin/
Disallow: /api/
Disallow: /_next/
Disallow: /private/
Disallow: /temp/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /favorites/
Disallow: /*/favorites/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /success/
Disallow: /*/success/
Disallow: /login/
Disallow: /*/login/
Disallow: /signup/
Disallow: /*/signup/
Disallow: /offline/
Disallow: /*/offline/
Disallow: /pwa-login/
Disallow: /*/pwa-login/
Disallow: /pdf-viewer/
Disallow: /*/pdf-viewer/
Disallow: /track/
Disallow: /*/track/
Disallow: /share/
Disallow: /*/share/
Disallow: /certificate/
Disallow: /*/certificate/

# Newsletter unsubscribe pages — token-gated, noindex in page metadata,
# but add an explicit Disallow here as well so we don't burn crawl budget
# on parameterized URLs (?token=...) that expand combinatorially.
Disallow: /newsletter/unsubscribe
Disallow: /ar/newsletter/unsubscribe
Disallow: /ru/newsletter/unsubscribe

# ============================================================
# SEARCH ENGINES
# ============================================================

User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /favorites/
Disallow: /*/favorites/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /success/
Disallow: /*/success/
Disallow: /login/
Disallow: /*/login/
Disallow: /signup/
Disallow: /*/signup/
Disallow: /offline/
Disallow: /*/offline/
Disallow: /newsletter/unsubscribe
Disallow: /ar/newsletter/unsubscribe
Disallow: /ru/newsletter/unsubscribe

# Google's additional crawlers for AI and indexing
User-agent: Google-Extended
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /favorites/
Disallow: /*/favorites/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /success/
Disallow: /*/success/
Disallow: /login/
Disallow: /*/login/
Disallow: /signup/
Disallow: /*/signup/
Disallow: /offline/
Disallow: /*/offline/

User-agent: GoogleOther
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /favorites/
Disallow: /*/favorites/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /success/
Disallow: /*/success/
Disallow: /login/
Disallow: /*/login/
Disallow: /signup/
Disallow: /*/signup/
Disallow: /offline/
Disallow: /*/offline/

User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /favorites/
Disallow: /*/favorites/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /success/
Disallow: /*/success/
Disallow: /login/
Disallow: /*/login/
Disallow: /signup/
Disallow: /*/signup/
Disallow: /offline/
Disallow: /*/offline/
Crawl-delay: 1

User-agent: Yandex
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /favorites/
Disallow: /*/favorites/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /success/
Disallow: /*/success/
Disallow: /login/
Disallow: /*/login/
Disallow: /signup/
Disallow: /*/signup/
Disallow: /offline/
Disallow: /*/offline/
Crawl-delay: 1

User-agent: Slurp
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /favorites/
Disallow: /*/favorites/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /success/
Disallow: /*/success/
Disallow: /login/
Disallow: /*/login/
Disallow: /signup/
Disallow: /*/signup/
Disallow: /offline/
Disallow: /*/offline/
Crawl-delay: 1

# Brave Search
User-agent: Bravebot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /favorites/
Disallow: /*/favorites/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /success/
Disallow: /*/success/
Disallow: /login/
Disallow: /*/login/
Disallow: /signup/
Disallow: /*/signup/
Disallow: /offline/
Disallow: /*/offline/

# ============================================================
# AI SEARCH & ASSISTANTS (drive citations and referral traffic)
# ============================================================

# OpenAI (ChatGPT search, training)
User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /checkout/
Disallow: /*/checkout/

# ChatGPT user-triggered browsing
User-agent: ChatGPT-User
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /checkout/
Disallow: /*/checkout/

# Perplexity AI search
User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /checkout/
Disallow: /*/checkout/

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /checkout/
Disallow: /*/checkout/

User-agent: anthropic-ai
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /checkout/
Disallow: /*/checkout/

# Apple (Siri, Apple Intelligence, Safari suggestions)
User-agent: Applebot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /favorites/
Disallow: /*/favorites/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /success/
Disallow: /*/success/
Disallow: /login/
Disallow: /*/login/
Disallow: /signup/
Disallow: /*/signup/
Disallow: /offline/
Disallow: /*/offline/

User-agent: Applebot-Extended
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /favorites/
Disallow: /*/favorites/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /success/
Disallow: /*/success/
Disallow: /login/
Disallow: /*/login/
Disallow: /signup/
Disallow: /*/signup/
Disallow: /offline/
Disallow: /*/offline/

# Amazon (Alexa, product recommendations)
User-agent: Amazonbot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /checkout/
Disallow: /*/checkout/

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /checkout/
Disallow: /*/checkout/

# Common Crawl (used by many AI systems)
User-agent: CCBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /checkout/
Disallow: /*/checkout/

# DeepSeek AI
User-agent: DeepSeekBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /favorites/
Disallow: /*/favorites/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /success/
Disallow: /*/success/

# Cohere AI
User-agent: cohere-ai
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /favorites/
Disallow: /*/favorites/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /success/
Disallow: /*/success/

# ============================================================
# SEO MONITORING TOOLS
# ============================================================

User-agent: AhrefsBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /favorites/
Disallow: /*/favorites/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /success/
Disallow: /*/success/
Disallow: /login/
Disallow: /*/login/
Disallow: /signup/
Disallow: /*/signup/
Disallow: /offline/
Disallow: /*/offline/
Crawl-delay: 2

User-agent: SemrushBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /profile/
Disallow: /*/profile/
Disallow: /cart/
Disallow: /*/cart/
Disallow: /favorites/
Disallow: /*/favorites/
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /success/
Disallow: /*/success/
Disallow: /login/
Disallow: /*/login/
Disallow: /signup/
Disallow: /*/signup/
Disallow: /offline/
Disallow: /*/offline/
Crawl-delay: 2

# ============================================================
# BLOCKED BOTS (scrapers with no value)
# ============================================================

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

# ============================================================
# AI CONTENT DISCOVERY (for LLMs and AI assistants)
# ============================================================

# Short index: https://genosys.ae/llms.txt
# Full index: https://genosys.ae/llms-full.txt
# Product index: https://genosys.ae/ai-products.txt

# Google Merchant Center Product Feed
# Feed: https://genosys.ae/feed/products.xml

# Blog discovery feeds
# RSS: https://genosys.ae/feed/blog.xml
# Atom: https://genosys.ae/feed/blog.atom