# Default: every crawler may fetch everything. A named group below overrides this
# one for that crawler, so each group repeats Allow: / rather than inheriting it.
User-agent: *
Allow: /

# Search engines. Googlebot and Bingbot build the indexes the site depends on, and
# Applebot powers Siri and Spotlight search.
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

# AI search and citation crawlers. These fetch pages so an answer engine can cite the
# site in a response, and are fetched on a user's behalf. Blocking them removes the
# site from AI answers, so they stay allowed.
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Training-only crawlers. These collect pages for model training and do not send
# readers back. They default to ALLOW. To opt out of training, change "Allow: /" to
# "Disallow: /" in the groups below; that does not affect search or citation above.
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

Sitemap: https://aitoolsmarketer.com/sitemap.xml