# ============================================
# ROBOTS.TXT - TMT (Target Mock Test)
# Search Engine Crawl Instructions
# ============================================

# ============================================
# 1. ALL CRAWLERS
# ============================================

User-agent: *

# ============================================
# 2. ALLOWED PATHS
# ============================================

# Main pages - Allow crawling
Allow: /
Allow: /index.php
Allow: /tests.php
Allow: /test.php
Allow: /result.php
Allow: /register.php
Allow: /login.php
Allow: /user-dashboard.php
Allow: /profile.php
Allow: /user-guide.php
Allow: /privacy-policy.php
Allow: /terms-and-conditions.php
Allow: /disclaimer.php
Allow: /about-us.php
Allow: /share.php

# Assets - Allow crawling
Allow: /assets/css/
Allow: /assets/js/
Allow: /assets/images/
Allow: /logo-icon/

# PWA Files - Allow crawling
Allow: /manifest.json
Allow: /sw.js

# SEO Files - Allow crawling
Allow: /robots.txt
Allow: /sitemap.xml

# ============================================
# 3. DISALLOWED PATHS
# ============================================

# Admin Panel - Disallow crawling
Disallow: /admin/

# API Endpoints - Disallow crawling
Disallow: /api/

# PHP Includes - Disallow crawling
Disallow: /includes/

# Uploads - Disallow crawling
Disallow: /uploads/

# Logs - Disallow crawling
Disallow: /logs/

# Config Files - Disallow crawling
Disallow: /config/
Disallow: /*.php$

# Temporary and system files
Disallow: /tmp/
Disallow: /temp/
Disallow: /backup/

# ============================================
# 4. SITEMAP LOCATION
# ============================================

Sitemap: https://tmt.edufavour.com/sitemap.xml

# ============================================
# 5. CRAWL DELAY (Respectful crawling)
# ============================================

Crawl-delay: 5

# ============================================
# 6. HOST (Optional - for search engines)
# ============================================

Host: https://tmt.edufavour.com

# ============================================
# 7. SPECIFIC BOTS
# ============================================

# Googlebot (Google Search)
User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /includes/
Disallow: /uploads/
Disallow: /logs/
Crawl-delay: 2

# Bingbot (Bing Search)
User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /includes/
Disallow: /uploads/
Disallow: /logs/
Crawl-delay: 2

# Yandex Bot
User-agent: YandexBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /includes/
Disallow: /uploads/
Disallow: /logs/
Crawl-delay: 3

# DuckDuckGo Bot
User-agent: DuckDuckBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /includes/
Disallow: /uploads/
Disallow: /logs/
Crawl-delay: 3

# Baidu Spider
User-agent: Baiduspider
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /includes/
Disallow: /uploads/
Disallow: /logs/
Crawl-delay: 5

# ============================================
# 8. BOTS TO BLOCK
# ============================================

# AI Bots (Prevent content scraping for AI training)
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

# Scrapers and bad bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: YandexImages
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Twitterbot
Disallow: /

# ============================================
# 9. SEO OPTIMIZATION NOTES
# ============================================

# Note: All public pages are allowed for indexing
# Admin and API endpoints are blocked for security
# Sitemap is available for better indexation
# Crawl delay ensures server doesn't get overloaded
# AI bots are blocked to protect content

# ============================================
# END OF ROBOTS.TXT
# ============================================