# Flies on the Visor — robots.txt
# Policy: Allow classic search engines and user-facing AI fetchers that cite sources.
#         Block AI training crawlers and AI training extensions.

# --- Standard search engines (full access) ---
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Yandex
Allow: /

User-agent: Applebot
Allow: /

# --- User-facing AI real-time fetchers (allowed; can cite/link) ---
# OpenAI Search bot for real-time fetching
User-agent: OAI-SearchBot
Allow: /

# ChatGPT browsing user agent (real-time fetches)
User-agent: ChatGPT-User
Allow: /

# Perplexity real-time fetcher
User-agent: PerplexityBot
Allow: /

User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

# --- Block model-training crawlers (no training on site content) ---
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

# Google Gemini/Vertex AI training & grounding control (does not affect Google Search)
User-agent: Google-Extended
Disallow: /

# Apple training extension (separate from Applebot search indexing)
User-agent: Applebot-Extended
Disallow: /

# ByteDance/ByteSpider (commonly used for model training)
User-agent: Bytespider
Disallow: /

# --- Housekeeping ---
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

Sitemap: https://fliesonthevisor.com/sitemap.html

Disallow: /*/1000$