# ─────────────────────────────────────────────────────────────────────────────
# Policy (2026-07-30):
#   ALLOW  — search/answer crawlers that can refer real users, including the
#            AI-search crawlers (Claude-SearchBot, Amzn-SearchBot) and the
#            AI-grounding opt-ins (Google-Extended, Applebot-Extended).
#   BLOCK  — pure AI *training* scrapers that send no traffic back.
#
# NOTE: robots.txt groups are NOT additive. A named User-agent group fully
# replaces the "*" group for that agent, so the shared Disallow list below is
# repeated verbatim in every allowed group. Keep them in sync when editing.
# ─────────────────────────────────────────────────────────────────────────────

# ── AI training crawlers — no access ─────────────────────────────────────────
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

# ── Allowed: AI search / answer crawlers (can refer users) ───────────────────
# Anthropic's search crawler. Distinct token from ClaudeBot/anthropic-ai above,
# which stay blocked — those are training crawlers.
User-agent: Claude-SearchBot
Disallow: /login
Disallow: /saved
Disallow: /settings
Disallow: /messages
Disallow: /chat
Disallow: /u/
Disallow: /projects/
Disallow: /my-projects
Disallow: /my-businesses
Disallow: /business/
Disallow: /api/
Disallow: /p/
Disallow: /b/

# Amazon's search crawler. Distinct token from Amazonbot above (still blocked).
User-agent: Amzn-SearchBot
Disallow: /login
Disallow: /saved
Disallow: /settings
Disallow: /messages
Disallow: /chat
Disallow: /u/
Disallow: /projects/
Disallow: /my-projects
Disallow: /my-businesses
Disallow: /business/
Disallow: /api/
Disallow: /p/
Disallow: /b/

# ── Allowed: Google ──────────────────────────────────────────────────────────
User-agent: Googlebot
Disallow: /login
Disallow: /saved
Disallow: /settings
Disallow: /messages
Disallow: /chat
Disallow: /u/
Disallow: /projects/
Disallow: /my-projects
Disallow: /my-businesses
Disallow: /business/
Disallow: /api/
Disallow: /p/
Disallow: /b/

# Google-Extended controls Gemini / Vertex grounding, NOT Search indexing.
# Unblocked 2026-07-30 to allow AI-surface grounding.
User-agent: Google-Extended
Disallow: /login
Disallow: /saved
Disallow: /settings
Disallow: /messages
Disallow: /chat
Disallow: /u/
Disallow: /projects/
Disallow: /my-projects
Disallow: /my-businesses
Disallow: /business/
Disallow: /api/
Disallow: /p/
Disallow: /b/

# ── Allowed: Microsoft ───────────────────────────────────────────────────────
User-agent: Bingbot
Disallow: /login
Disallow: /saved
Disallow: /settings
Disallow: /messages
Disallow: /chat
Disallow: /u/
Disallow: /projects/
Disallow: /my-projects
Disallow: /my-businesses
Disallow: /business/
Disallow: /api/
Disallow: /p/
Disallow: /b/
Crawl-delay: 5

User-agent: msnbot
Disallow: /login
Disallow: /saved
Disallow: /settings
Disallow: /messages
Disallow: /chat
Disallow: /u/
Disallow: /projects/
Disallow: /my-projects
Disallow: /my-businesses
Disallow: /business/
Disallow: /api/
Disallow: /p/
Disallow: /b/
Crawl-delay: 5

# ── Allowed: Apple ───────────────────────────────────────────────────────────
User-agent: Applebot
Disallow: /login
Disallow: /saved
Disallow: /settings
Disallow: /messages
Disallow: /chat
Disallow: /u/
Disallow: /projects/
Disallow: /my-projects
Disallow: /my-businesses
Disallow: /business/
Disallow: /api/
Disallow: /p/
Disallow: /b/

# Applebot-Extended controls Apple Intelligence training.
# Unblocked 2026-07-30 alongside Google-Extended.
User-agent: Applebot-Extended
Disallow: /login
Disallow: /saved
Disallow: /settings
Disallow: /messages
Disallow: /chat
Disallow: /u/
Disallow: /projects/
Disallow: /my-projects
Disallow: /my-businesses
Disallow: /business/
Disallow: /api/
Disallow: /p/
Disallow: /b/

# ── PetalBot (Huawei/Petal Search) — low-value, heavily rate-limited ─────────
# Honors robots.txt + Crawl-delay. Switch to "Disallow: /" to drop it entirely.
User-agent: PetalBot
Disallow: /login
Disallow: /saved
Disallow: /settings
Disallow: /messages
Disallow: /chat
Disallow: /u/
Disallow: /projects/
Disallow: /my-projects
Disallow: /my-businesses
Disallow: /business/
Disallow: /api/
Disallow: /p/
Disallow: /b/
Crawl-delay: 10

# ── All other crawlers ───────────────────────────────────────────────────────
User-agent: *
Disallow: /login
Disallow: /saved
Disallow: /settings
Disallow: /messages
Disallow: /chat
Disallow: /u/
Disallow: /projects/
Disallow: /my-projects
Disallow: /my-businesses
Disallow: /business/
Disallow: /api/
Disallow: /p/
Disallow: /b/

Sitemap: https://api.linkees.com/api/v1/sitemaps/sitemap.xml