# ============================================================================
# ArcaneWorld.net - Robots.txt Configuration
# Purpose: Control search engine crawler access to website resources
# Last Updated: November 2025
# ============================================================================

# ==============================================================================
# SECTION 1: DEFAULT RULES (All Search Engines)
# ==============================================================================

User-agent: *

# ALLOW: Public pages that should be indexed
# Homepage and informational pages
Allow: /
Allow: /*/
Allow: /*/core
Allow: /*/main
Allow: /*/wolf
Allow: /*/about
Allow: /*/download
Allow: /*/features
Allow: /*/patches
Allow: /*/news
Allow: /*/media
Allow: /*/streamers
Allow: /*/term
Allow: /*/policy
Allow: /*/privacy
Allow: /*/agreement
Allow: /*/eula
Allow: /*/disclaimer
Allow: /*/server_rules
Allow: /*/start
Allow: /*/how_to_connect
Allow: /*/how_to_donate

# ==============================================================================
# SECTION 2: DISALLOW - PRIVATE/INTERNAL PAGES (Must NOT be indexed)
# ==============================================================================

# Admin panel - STRICTLY CONFIDENTIAL
Disallow: /backend_*
Disallow: */backend_*

# User Cabinet - Private User Data
Disallow: /*/cabinet/
Disallow: */cabinet/*

# Authentication pages
Disallow: /*/login
Disallow: /*/login_2fa
Disallow: /*/register
Disallow: /*/auth/

# API Endpoints - Dynamic Data, Not for Indexing
Disallow: /api/
Disallow: */api/*

# System files and configurations
Disallow: /.env
Disallow: /.git
Disallow: /.gitignore
Disallow: /composer.json
Disallow: /composer.lock
Disallow: /package.json
Disallow: /phpunit.xml

# Laravel System Directories
Disallow: /vendor/
Disallow: /storage/
Disallow: /bootstrap/cache/
Disallow: /database/
Disallow: /node_modules/
Disallow: /resources/

# Temporary and Cache Files
Disallow: /tmp/
Disallow: /cache/
Disallow: *.log
Disallow: *.sql
Disallow: *.zip
Disallow: *.rar

# ==============================================================================
# SECTION 3: CRAWL DELAY & REQUEST RATE
# ==============================================================================

# Standard crawl delay (in seconds)
Crawl-delay: 1

# ==============================================================================
# SECTION 4: GOOGLE-SPECIFIC RULES
# ==============================================================================

User-agent: Googlebot
Allow: /
Allow: /*/
Crawl-delay: 0.5

# Google can index sitemap
Allow: /sitemap.xml
Allow: /sitemap_*.xml

# ==============================================================================
# SECTION 5: YANDEX-SPECIFIC RULES (Russian Search Engine)
# ==============================================================================

User-agent: YandexBot
Disallow: /*/cabinet/
Disallow: /backend_*
Crawl-delay: 1

# ==============================================================================
# SECTION 6: BLOCKING KNOWN HARMFUL BOTS
# ==============================================================================

# Aggressive scrapers and malicious bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: SputnikImageBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# ==============================================================================
# SECTION 7: MEDIA & CDN RULES
# ==============================================================================

# Allow search engines to index images and media
User-agent: Googlebot-Image
Allow: /images/
Allow: /img/
Allow: /assets/

User-agent: *
Allow: /*.webp
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.gif
Allow: /*.svg

# ==============================================================================
# SECTION 8: SITEMAP LOCATIONS
# ==============================================================================

# Sitemaps for better indexing
Sitemap: https://arcaneworld.net/sitemap.xml
Sitemap: https://arcaneworld.net/sitemap_articles.xml
Sitemap: https://arcaneworld.net/sitemap_pages.xml