# -------------------------------------
# MARKSKINEXPERT.COM CRAWLER POLICY
# -------------------------------------

# --- Allow these trusted crawlers ---
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Internet Archive
User-agent: ia_archiver
Allow: /

User-agent: archive.org_bot
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# UK Sovereign
User-agent: MojeekBot
Allow: /

# EU Sovereign
User-agent: Qwantify
Allow: /

# Central Europe
User-agent: SeznamBot
Allow: /

# Google / Gemini
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Googlebot-Mobile
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /

User-agent: AdsBot-Google-Mobile-Apps
Allow: /

User-agent: Mediapartners-Google
Allow: /

# Bing
User-agent: bingbot
Allow: /

User-agent: BingPreview
Allow: /

User-agent: adidxbot
Allow: /

# SEO / discovery
User-agent: AhrefsBot
Allow: /

User-agent: PerplexityBot
Allow: /

# --- Explicitly disallow known AI/data crawlers ---
User-agent: Baiduspider
Disallow: /

User-agent: Baiduspider-image
Disallow: /

User-agent: Baiduspider-video
Disallow: /

User-agent: Baiduspider-news
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: DeepSeekBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: cohere-ai
Disallow: /

# Allow Amazonbot only for sections containing affiliate/product content
User-agent: Amazonbot
Allow: /blog/
Allow: /reviews/
Allow: /ingredient-directory/
Disallow: /

User-agent: Amazonbot-Image
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# --- Default: block everything else ---
User-agent: *
Disallow: /

# -------------------------------------
# LEGAL NOTICE
# -------------------------------------
# This site and its content are NOT licensed for AI model training,
# dataset compilation, or automated scraping. Unauthorized use of
# any content for such purposes constitutes copyright infringement.

# -------------------------------------
# SITEMAP INDEX
# -------------------------------------
Sitemap: https://markskinexpert.com/sitemap_index.xml