# invisit.dev welcomes search engines, AI assistants and the crawlers that train them.
# Every page is also Markdown: add .md to its address, or send Accept: text/markdown.
# Index for language models: https://invisit.dev/llms.txt

# One group for every crawler: a crawler named in a group of its own ignores the * group, so
# the AI crawlers are listed here, by name, under the same rules as everyone else.
User-agent: *
User-agent: Googlebot
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Google-CloudVertexBot
User-agent: Bingbot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: ClaudeBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: meta-webindexer
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: Amazonbot
User-agent: Amzn-SearchBot
User-agent: Amzn-User
User-agent: DuckAssistBot
User-agent: DuckDuckBot
User-agent: MistralAI-Index
User-agent: MistralAI-User
User-agent: MistralAI-Training
User-agent: CCBot
User-agent: ShapBot
User-agent: YouBot
User-agent: Kagibot
User-agent: cohere-ai
User-agent: GrokBot
User-agent: xAI-SearchBot
User-agent: Bytespider
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
# The docs search index: binary chunks, nothing to read.
Disallow: /pagefind/

Sitemap: https://invisit.dev/sitemap.xml
Agentmap: https://invisit.dev/.well-known/ai-catalog.json