# IIS Rescue — robots.txt
# Full crawling is welcome. The named groups below address AI crawlers and
# AI-usage control tokens explicitly rather than relying on the wildcard, which
# some operators treat conservatively.
#
# IMPORTANT: a crawler matching a named group ignores the wildcard group
# entirely, so "Disallow: /downloads/" is repeated in every group. /downloads/
# holds the gated migration guide and stays closed to all crawlers.
#
# Disallow is listed before Allow in each group deliberately. Google and Bing
# resolve conflicts by longest match, but simpler crawlers take the first match;
# this ordering keeps /downloads/ closed under either interpretation.
#
# /thanks.html, /download.html and /privacy.html are deliberately NOT disallowed
# here. Each carries <meta name="robots" content="noindex, follow">; a crawler
# blocked in robots.txt would never fetch the page to see that directive.

User-agent: *
Disallow: /downloads/
Allow: /

# --- OpenAI: training crawler, search index, user-initiated fetches ---
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
Disallow: /downloads/
Allow: /

# --- Anthropic ---
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
Disallow: /downloads/
Allow: /

# --- Perplexity ---
User-agent: PerplexityBot
User-agent: Perplexity-User
Disallow: /downloads/
Allow: /

# --- Microsoft Bing / Copilot ---
User-agent: Bingbot
Disallow: /downloads/
Allow: /

# --- Common Crawl (feeds many model training corpora) ---
User-agent: CCBot
Disallow: /downloads/
Allow: /

# --- Apple (Siri / Spotlight). Applebot is the crawler and is named separately
# --- from Applebot-Extended below, which is only a usage-control token: naming
# --- one does not cover the other.
User-agent: Applebot
Disallow: /downloads/
Allow: /

# --- Amazon (Alexa / Rufus grounding) ---
User-agent: Amazonbot
Disallow: /downloads/
Allow: /

# --- Meta (Meta AI grounding) ---
User-agent: meta-externalagent
Disallow: /downloads/
Allow: /

# --- DuckDuckGo (DuckAssist answers) ---
User-agent: DuckAssistBot
Disallow: /downloads/
Allow: /

# --- AI-usage control tokens. Not crawlers: these govern whether already
# --- crawled content may be used for AI grounding/training. Allow = opt in.
User-agent: Google-Extended
User-agent: Applebot-Extended
Disallow: /downloads/
Allow: /

# --- Bytespider (ByteDance/TikTok). Blocked deliberately: it crawls
# --- aggressively and feeds no assistant that a US small-business buyer would
# --- ask "who do I hire to migrate my IIS site". No citation upside, real
# --- crawl cost. This is the one named group that is not an opt-in.
User-agent: Bytespider
Disallow: /

Sitemap: https://iisrescue.com/sitemap.xml