# https://www.aknmimm.com/robots.txt
#
# Path matching is prefix-based and anchored at the start of the path, so "/dashboard"
# blocks both /dashboard and /dashboard/anything. The earlier version of this file wrote
# "/dashboard/" (with a trailing slash), which does NOT match the actual route "/dashboard"
# and left every signed-in app screen crawlable.
#
# "/analytics" does not affect the marketing page "/features/analytics", because matching
# is anchored at the start of the path.

# ---------------------------------------------------------------------------
# Content-Signal: machine-readable AI preferences (contentsignals.org).
# The marketing site and blog are open for search and AI-assistant citation
# (search=yes, ai-input=yes) but closed to model training (ai-train=no). The
# training-only crawlers below (GPTBot, anthropic-ai, CCBot, Bytespider) are
# blocked to match, while search/grounding bots stay allowed.
# ---------------------------------------------------------------------------
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: *
Allow: /
Disallow: /api
Disallow: /auth
Disallow: /dashboard
Disallow: /videos
Disallow: /schedule
Disallow: /analytics
Disallow: /comments
Disallow: /settings
Disallow: /inbox
Disallow: /billing
Disallow: /revenue
Disallow: /ai
Disallow: /ab-tests
Disallow: /cdn-cgi/

# ---------------------------------------------------------------------------
# AI answer engines.
#
# A user-agent group REPLACES the "*" group for that bot rather than merging with it, so
# each bot listed here repeats the full disallow list. Any bot not listed still falls back
# to "*" above and is allowed.
#
# These are spelled out so the intent is unambiguous: the marketing site and blog are open
# to AI crawlers for retrieval and citation; only the signed-in app is closed.
# ---------------------------------------------------------------------------

# OpenAI: OAI-SearchBot surfaces pages in ChatGPT search results, ChatGPT-User handles
# user-initiated browsing. GPTBot is the training crawler; per the ai-train=no policy it
# is split out below and blocked.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
Allow: /
Disallow: /api
Disallow: /auth
Disallow: /dashboard
Disallow: /videos
Disallow: /schedule
Disallow: /analytics
Disallow: /comments
Disallow: /settings
Disallow: /inbox
Disallow: /billing
Disallow: /revenue
Disallow: /ai
Disallow: /ab-tests
Disallow: /cdn-cgi/

# GPTBot trains models on covered content. The Content-Signal above declares
# ai-train=no, so it is blocked outright.
User-agent: GPTBot
Disallow: /

# Anthropic (Claude). Claude-Web is the browsing bot behind quote-worthy chat interactions.
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Claude-Web
Allow: /
Disallow: /api
Disallow: /auth
Disallow: /dashboard
Disallow: /videos
Disallow: /schedule
Disallow: /analytics
Disallow: /comments
Disallow: /settings
Disallow: /inbox
Disallow: /billing
Disallow: /revenue
Disallow: /ai
Disallow: /ab-tests
Disallow: /cdn-cgi/

# anthropic-ai runs Anthropic's background/automation processes over retrieved content,
# which falls under training. Blocked to match the ai-train=no policy.
User-agent: anthropic-ai
Disallow: /

# Perplexity.
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /
Disallow: /api
Disallow: /auth
Disallow: /dashboard
Disallow: /videos
Disallow: /schedule
Disallow: /analytics
Disallow: /comments
Disallow: /settings
Disallow: /inbox
Disallow: /billing
Disallow: /revenue
Disallow: /ai
Disallow: /ab-tests
Disallow: /cdn-cgi/

# Google-Extended does not affect Google Search ranking or indexing. It controls whether
# the site can be used to ground and train Gemini, which is what makes the blog eligible
# for citation in Gemini answers.
User-agent: Google-Extended
Allow: /
Disallow: /api
Disallow: /auth
Disallow: /dashboard
Disallow: /videos
Disallow: /schedule
Disallow: /analytics
Disallow: /comments
Disallow: /settings
Disallow: /inbox
Disallow: /billing
Disallow: /revenue
Disallow: /ai
Disallow: /ab-tests
Disallow: /cdn-cgi/

# Apple Intelligence / Siri.
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /
Disallow: /api
Disallow: /auth
Disallow: /dashboard
Disallow: /videos
Disallow: /schedule
Disallow: /analytics
Disallow: /comments
Disallow: /settings
Disallow: /inbox
Disallow: /billing
Disallow: /revenue
Disallow: /ai
Disallow: /ab-tests
Disallow: /cdn-cgi/

# Common Crawl feeds a large share of open model training corpora. per the ai-train=no
# policy the site opts out of training, so CCBot is blocked entirely.
User-agent: CCBot
Disallow: /

# ByteDance (TikTok) crawler. Its primary use is training AI retrieval and
# recommendation models, so it is blocked per the ai-train=no policy.
User-agent: Bytespider
Disallow: /

# Amazonbot crawls for Amazon's generative search (Rufus) and shopping answers.
User-agent: Amazonbot
Allow: /
Disallow: /api
Disallow: /auth
Disallow: /dashboard
Disallow: /videos
Disallow: /schedule
Disallow: /analytics
Disallow: /comments
Disallow: /settings
Disallow: /inbox
Disallow: /billing
Disallow: /revenue
Disallow: /ai
Disallow: /ab-tests
Disallow: /cdn-cgi/

# Meta AI: Meta-ExternalAgent is the standalone crawler that surfaces pages in
# AI answers outside Facebook/Messenger surfaces.
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api
Disallow: /auth
Disallow: /dashboard
Disallow: /videos
Disallow: /schedule
Disallow: /analytics
Disallow: /comments
Disallow: /settings
Disallow: /inbox
Disallow: /billing
Disallow: /revenue
Disallow: /ai
Disallow: /ab-tests
Disallow: /cdn-cgi/

Sitemap: https://www.aknmimm.com/sitemap.xml