robots.txt

# robots.txt for The American Whistle Corporation
# AI crawler directives — AWC WANTS to be cited in AI answers about
# Made-in-USA whistles, government procurement, Buy American Act
# eligible suppliers. ALLOW posture across the board for retrieval +
# training crawlers, with /cart, /checkouts, /account, /search
# DISALLOWED to keep transactional surfaces out of AI corpora.
#
# Generated 2026-05-07 by Aelo (BOS AEO/GEO agent) per owner feedback.

# Preferred domain — non-www redirects to www.
Host: www.americanwhistle.com

# OpenAI training crawler
User-agent: GPTBot
Allow: /collections/
Allow: /products/
Allow: /pages/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /agents.md
Allow: /sitemap.xml
Disallow: /cart
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts/
Disallow: /account
Disallow: /search
Disallow: /?q=
Crawl-delay: 5

# OpenAI live-browsing on user request
User-agent: ChatGPT-User
Allow: /collections/
Allow: /products/
Allow: /pages/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /agents.md
Allow: /sitemap.xml
Disallow: /cart
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts/
Disallow: /account
Disallow: /search
Disallow: /?q=
Crawl-delay: 5

# OpenAI ChatGPT search-style retrieval
User-agent: OAI-SearchBot
Allow: /collections/
Allow: /products/
Allow: /pages/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /agents.md
Allow: /sitemap.xml
Disallow: /cart
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts/
Disallow: /account
Disallow: /search
Disallow: /?q=
Crawl-delay: 5

# Google Gemini training opt-in/opt-out
User-agent: Google-Extended
Allow: /collections/
Allow: /products/
Allow: /pages/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /agents.md
Allow: /sitemap.xml
Disallow: /cart
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts/
Disallow: /account
Disallow: /search
Disallow: /?q=
Crawl-delay: 5

# Common Crawl — training data for many open models
User-agent: CCBot
Allow: /collections/
Allow: /products/
Allow: /pages/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /agents.md
Allow: /sitemap.xml
Disallow: /cart
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts/
Disallow: /account
Disallow: /search
Disallow: /?q=
Crawl-delay: 5

# Anthropic Claude training crawler
User-agent: ClaudeBot
Allow: /collections/
Allow: /products/
Allow: /pages/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /agents.md
Allow: /sitemap.xml
Disallow: /cart
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts/
Disallow: /account
Disallow: /search
Disallow: /?q=
Crawl-delay: 5

# Anthropic Claude live-browsing on user request
User-agent: Claude-Web
Allow: /collections/
Allow: /products/
Allow: /pages/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /agents.md
Allow: /sitemap.xml
Disallow: /cart
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts/
Disallow: /account
Disallow: /search
Disallow: /?q=
Crawl-delay: 5

# Perplexity AI retrieval
User-agent: PerplexityBot
Allow: /collections/
Allow: /products/
Allow: /pages/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /agents.md
Allow: /sitemap.xml
Disallow: /cart
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts/
Disallow: /account
Disallow: /search
Disallow: /?q=
Crawl-delay: 5

# Apple Intelligence training opt-in/opt-out
User-agent: Applebot-Extended
Allow: /collections/
Allow: /products/
Allow: /pages/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /agents.md
Allow: /sitemap.xml
Disallow: /cart
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts/
Disallow: /account
Disallow: /search
Disallow: /?q=
Crawl-delay: 5

# ByteDance/Doubao training crawler
User-agent: Bytespider
Allow: /collections/
Allow: /products/
Allow: /pages/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /agents.md
Allow: /sitemap.xml
Disallow: /cart
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts/
Disallow: /account
Disallow: /search
Disallow: /?q=
Crawl-delay: 5

# Amazon Alexa / Q retrieval
User-agent: Amazonbot
Allow: /collections/
Allow: /products/
Allow: /pages/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /agents.md
Allow: /sitemap.xml
Disallow: /cart
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts/
Disallow: /account
Disallow: /search
Disallow: /?q=
Crawl-delay: 5

# Meta AI retrieval
User-agent: Meta-ExternalAgent
Allow: /collections/
Allow: /products/
Allow: /pages/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /agents.md
Allow: /sitemap.xml
Disallow: /cart
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts/
Disallow: /account
Disallow: /search
Disallow: /?q=
Crawl-delay: 5

# DiffBot AI knowledge graph crawler
User-agent: DiffBot
Allow: /collections/
Allow: /products/
Allow: /pages/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /agents.md
Allow: /sitemap.xml
Disallow: /cart
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts/
Disallow: /account
Disallow: /search
Disallow: /?q=
Crawl-delay: 5

# Default for all other crawlers (Googlebot, Bingbot, etc. — Shopify default applies)
User-agent: *
Disallow: /cart
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts/
Disallow: /account
Disallow: /search
Disallow: /?q=

Sitemap: https://www.americanwhistle.com/sitemap.xml