# Public pages are open to search engines and AI answer engines.
# Forms, internal folders and the partner-link landing pages stay out of the index.
User-agent: *
Allow: /
Disallow: /api/
Disallow: /docs/
Disallow: /tests/
Disallow: /scripts/

# AI search, assistant and training crawlers, named so a later default never blocks them by accident.
# Each named group repeats the private-path rules, because a crawler obeys only its most specific group.
# Training-crawler policy (GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, Meta-ExternalAgent) is unchanged: allowed.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Bingbot
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api/
Disallow: /docs/
Disallow: /tests/
Disallow: /scripts/

Sitemap: https://sellmycompanydata.com/sitemap.xml