User-agent: *
Allow: /
# No bot-specific rules below: GPTBot, ChatGPT-User, OAI-SearchBot, PerplexityBot,
# ClaudeBot, anthropic-ai, Google-Extended, Bingbot, and CCBot all inherit this same
# permissive rule intentionally, to maximize AI-answer-engine citation of
# public marketing pages. Do not add a narrower block for these unless there
# is a specific reason to opt a given crawler out.
Disallow: /admin/
Disallow: /kai/
# /portal is deliberately NOT disallowed (2026-08-17). `Disallow: /portal` is a
# PREFIX match, so it also covered the legacy /portal/about.html,
# /portal/pricing.html, /portal/terms.html and /portal/Demo/?v= URLs that
# existed before the 2026-07-31 static-path rename (when static/ was mounted at
# /portal instead of root). Those paths now correctly 404 — but a blocked URL
# cannot be crawled, so Google could never SEE the 404 and could never drop the
# stale entries: Search Console reported them as "Indexed, though blocked by
# robots.txt" and the removal validation failed on 8/11/26 for exactly this
# reason. Same trap already documented below for terms.html/privacy.html.
# The SPA itself (/portal and /portal/, the only two real routes — every other
# /portal/* path 404s) is kept out of the index by
# <meta name="robots" content="noindex,follow"> in static/portal-app.html,
# which Google can only act on if it is allowed to fetch the page.
# Do NOT re-add a Disallow here unless these URLs are first confirmed dropped
# from the index in Search Console.
# static/Demo/ contains exactly one file (index.html), and both /Demo/ and
# /Demo/index.html serve it (StaticFiles html=True). A Disallow here protected
# nothing and risked a bare-URL listing for /Demo/ — blocked URLs can still be
# indexed from inbound links, and a blocked crawl can never read the canonical.
# Both URLs stay crawlable; the page carries <meta name="robots"
# content="noindex,follow"> to keep it out of the search index (same reasoning
# as terms.html/privacy.html below — a Disallow would block the crawl and
# Google would never read the noindex, leaving it eligible for a bare-URL
# listing from inbound links). It is also no longer listed in sitemap.xml.
# The self-referential canonical (-> /Demo/index.html) still consolidates the
# /Demo/, /Demo/index.html and /Demo/?v=<ts> cache-buster variants.
# Do NOT add a Disallow here unless the page is first confirmed dropped from
# the index in Search Console.
Allow: /Demo/
Disallow: /checkout.html
Disallow: /custom-checkout.html
Disallow: /billing.html
Disallow: /dashboard.html
Disallow: /launcher.html
Disallow: /portal-app.html
Disallow: /success.html
Disallow: /unsubscribed.html
Disallow: /unsubscribe.html
Disallow: /api/
# terms.html/privacy.html/aup.html/dpa.html are kept OUT of the search index,
# but deliberately still crawlable. They carry <meta name="robots"
# content="noindex,follow"> instead of a Disallow here, because a Disallow
# would block the crawl and Google would therefore never read the noindex —
# leaving them eligible to appear as bare-URL listings via the footer links on
# every page. "follow" keeps footer link equity flowing through them.
# Do NOT add Disallow lines for these unless the pages have first been
# confirmed dropped from the index in Search Console.

Sitemap: https://anweshna.com/sitemap.xml