# Njoer Robots.txt Configuration
# Updated for 2026 Web Standards & AI Compatibility

Sitemap: https://njoer.com/sitemap.xml

# -----------------------------------------------------------------------------
# General Search Engines & Standard Crawlers
# -----------------------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /control/
Disallow: /control_panel/
Disallow: /auth/
Disallow: /sign-in
Disallow: /register
Disallow: /dev/
Disallow: /health
Disallow: /mailbox/

# -----------------------------------------------------------------------------
# AI & LLM Crawlers - Visibility Settings
# -----------------------------------------------------------------------------

# OpenAI (ChatGPT Search & Training)
# GPTBot is used for training and retrieval. ChatGPT-User is for live browsing.
User-agent: GPTBot
Allow: /
Disallow: /control/
Disallow: /auth/

User-agent: ChatGPT-User
Allow: /
Disallow: /control/
Disallow: /auth/

# Google AI (Gemini / Vertex AI)
# Google-Extended controls usage for AI training (distinct from Googlebot)
User-agent: Google-Extended
Allow: /
Disallow: /control/
Disallow: /auth/

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /
Disallow: /control/
Disallow: /auth/

# Perplexity AI (Answer Engine)
User-agent: PerplexityBot
Allow: /
Disallow: /control/
Disallow: /auth/

# Meta / Facebook AI
User-agent: FacebookBot
Allow: /
Disallow: /control/
Disallow: /auth/

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /
Disallow: /control/
Disallow: /auth/

# Amazon Q / Alexa
User-agent: Amazonbot
Allow: /
Disallow: /control/
Disallow: /auth/

# -----------------------------------------------------------------------------
# Bulk Scrapers / Training Datasets (Optional Blocks)
# -----------------------------------------------------------------------------

# Common Crawl (Used for training most base models)
# Uncomment the following lines to block bulk training data collection:
# User-agent: CCBot
# Disallow: /

# User-agent: anthropic-ai
# Disallow: /

# User-agent: Bytespider
# Disallow: /