# Search + AI crawlers we explicitly want.
#
# These are the agents that gate whether we can be RETRIEVED and CITED — not the
# training-only bots. Getting this list wrong is common:
#   - OAI-SearchBot gates ChatGPT search answers. Per OpenAI's docs, sites opted
#     out of it "will not be shown in ChatGPT search answers". GPTBot is
#     training-only and does NOT control citation.
#   - Claude-SearchBot / Claude-User are the retrieval agents; ClaudeBot is
#     training-only.
#   - Googlebot is the AI Overviews control. Google-Extended is NOT — Google's
#     own docs say it "does not impact a site's inclusion in Google Search".
#     It's listed below only because it costs nothing to permit.
#   - Bingbot matters twice over: Bing is the default engine on Windows (our
#     platform) and is the retrieval substrate behind ChatGPT search and Copilot.
#
# IMPORTANT: robots.txt groups are NOT additive. The most specific matching
# group wins outright, so this group must repeat every Disallow from the
# wildcard group below — otherwise these agents would be free to crawl /api/
# and /auth/.
User-agent: Googlebot
User-agent: Bingbot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
Allow: /
Disallow: /auth/
Disallow: /billing/
Disallow: /checkout/
Disallow: /team/accept
Disallow: /blog-post-template/
Disallow: /play/
Disallow: /api/

User-agent: *
Allow: /
Disallow: /auth/
Disallow: /billing/
Disallow: /checkout/
Disallow: /team/accept
Disallow: /blog-post-template/
# Store-review supporting material (the Play foreground-service declaration).
# Not a page for readers, and indexing it would put an internal justification
# document in front of searchers.
Disallow: /play/
# Counted download endpoint — crawling it inflates download stats and burns
# KV writes on the namespace that also serves the blog. Humans reach it from
# the buttons; bots have no reason to.
Disallow: /api/

# NOTE: Crawl-delay was removed 2026-08-16. Google ignores it, but Bing HONORS
# it — and at the time we had zero pages in Bing's index (`url:pithflow.com`
# returned nothing, even for the brand name). Throttling the one crawler we
# most needed was working against us. Do not reintroduce it.

# Auto-generated by @astrojs/sitemap at build time.
# Includes hreflang annotations for en/es/pt/fr/de/ja.
Sitemap: https://pithflow.com/sitemap-index.xml

# KV-ingested blog posts (jervis blog automation) — served by a Pages
# Function, not present in the build-time sitemap above.
Sitemap: https://pithflow.com/blog-sitemap.xml