# ===========================================
# AlgoVesta Robots.txt v3.3
# https://algovesta.com/robots.txt
# Last Updated: 2026-08-17
# 
# SEO + AEO + GEO Optimized Configuration
# Supports: Google, Bing, Yandex, Baidu, Naver
# AI Engines: GPT, Gemini, Claude, Perplexity
# ===========================================

# ===========================================
# GENERAL CRAWLERS (Google, Bing, Yahoo, etc.)
# ===========================================
User-agent: *

# Allow all public pages
Allow: /

# Allow CSS, JS, Images for rendering (Core Web Vitals)
Allow: /css/
Allow: /js/
Allow: /images/
Allow: /assets/
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.svg$
Allow: /*.webp$

# Disallow user-specific/dashboard pages (no SEO value)
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html

# Disallow auth pages (no index value)
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html

# Disallow API endpoints and backend
Disallow: /api/
Disallow: /backend/

# Disallow non-content files and internal/underscore directories
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# Crawl-delay for politeness
Crawl-delay: 1

# ===========================================
# GOOGLE SPECIFIC CRAWLER
# ===========================================
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /backend/
Disallow: /profile.html
Disallow: /checkout.html
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

User-agent: Googlebot-Image
Allow: /images/
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /api/
Disallow: /backend/
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

User-agent: Googlebot-Video
Allow: /
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /api/
Disallow: /backend/
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# ===========================================
# BING SPECIFIC CRAWLER
# ===========================================
User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /backend/
Disallow: /profile.html
Crawl-delay: 2
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# ===========================================
# YANDEX (Russia)
# ===========================================
User-agent: Yandex
Allow: /
Disallow: /api/
Disallow: /backend/
Disallow: /profile.html
Crawl-delay: 2
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# ===========================================
# BAIDU (China)
# ===========================================
User-agent: Baiduspider
Allow: /
Disallow: /api/
Disallow: /backend/
Disallow: /profile.html
Crawl-delay: 3
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# ===========================================
# NAVER (Korea)
# ===========================================
User-agent: Yeti
Allow: /
Disallow: /api/
Disallow: /backend/
Crawl-delay: 2
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# ===========================================
# AI CRAWLERS (GEO/AEO Optimization)
# ===========================================

# OpenAI GPTBot (ChatGPT, GPT-4)
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /backend/
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /data-central.html
Disallow: /checkout.html
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# OpenAI ChatGPT-User (Web browsing)
User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /backend/
Disallow: /profile.html
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# Google Bard / Gemini
User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /backend/
Disallow: /profile.html
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# Anthropic Claude
User-agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /backend/
Disallow: /profile.html
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /backend/
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# Anthropic Claude (user-triggered fetch — Claude.ai/Claude Code citations)
User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /backend/
Disallow: /profile.html
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# Perplexity AI (crawler + user-triggered fetch)
User-agent: PerplexityBot
Allow: /
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /api/
Disallow: /backend/
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /backend/
Disallow: /profile.html
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# Common Crawl (used by many AI training datasets)
User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /backend/
Disallow: /profile.html
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# Cohere AI
User-agent: cohere-ai
Allow: /
Disallow: /api/
Disallow: /backend/
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# Meta AI (Facebook)
User-agent: FacebookBot
Allow: /
Disallow: /api/
Disallow: /backend/
Disallow: /profile.html
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

User-agent: facebookexternalhit
Allow: /
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /api/
Disallow: /backend/
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# Apple Bot (Siri, Spotlight)
User-agent: Applebot
Allow: /
Disallow: /api/
Disallow: /backend/
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# You.com AI
User-agent: YouBot
Allow: /
Disallow: /api/
Disallow: /backend/
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# OpenAI Search (ChatGPT search citations)
User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /backend/
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# Anthropic Claude (legacy web fetcher)
User-agent: Claude-Web
Allow: /
Disallow: /api/
Disallow: /backend/
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# Apple Intelligence / Siri (AI)
User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /backend/
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# Amazon (Alexa / AI)
User-agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /backend/
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# Meta AI (Llama)
User-agent: meta-externalagent
Allow: /
Disallow: /api/
Disallow: /backend/
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

User-agent: meta-externalfetcher
Allow: /
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /api/
Disallow: /backend/
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# Mistral AI
User-agent: MistralAI-User
Allow: /
Disallow: /api/
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /backend/
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# DuckDuckGo AI Assist
User-agent: DuckAssistBot
Allow: /
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /api/
Disallow: /backend/
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# ByteDance / Doubao AI — gorunurluk icin acik (robots.txt'i her zaman saymadigi
# bilinir; gercek sinirlama WAF tarafinda yapilir)
User-agent: Bytespider
Allow: /
Disallow: /api/
Disallow: /backend/
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# NOT: DeepSeek resmi crawler UA yayinlamiyor; egitim verisini buyuk olcude
# Common Crawl (CCBot — yukarida Allow) uzerinden alir.

# ===========================================
# SEO ANALYSIS CRAWLERS
# ===========================================

# Ahrefs
User-agent: AhrefsBot
Allow: /
Crawl-delay: 5
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /api/
Disallow: /backend/
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# SEMrush
User-agent: SemrushBot
Allow: /
Crawl-delay: 5
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /api/
Disallow: /backend/
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# Majestic (link index)
User-agent: MJ12bot
Allow: /
Crawl-delay: 10
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /api/
Disallow: /backend/
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# Screaming Frog
User-agent: Screaming Frog SEO Spider
Allow: /
# --- ortak koruma (#548): bu grup '*' blogunu YOK SAYAR, set burada tekrarlanir
Disallow: /profile.html
Disallow: /analytics-pro.html
Disallow: /api-keys-new.html
Disallow: /app.html
Disallow: /data-central.html
Disallow: /affiliate-dashboard.html
Disallow: /mcp-authorize.html
Disallow: /support-ticket.html
Disallow: /checkout.html
Disallow: /payment-success.html
Disallow: /payment-cancel.html
Disallow: /login.html
Disallow: /forgot-password.html
Disallow: /reset-password.html
Disallow: /verify-email.html
Disallow: /api/
Disallow: /backend/
Disallow: /_*/
Disallow: /cdn-cgi/
Disallow: /*.log$
Disallow: /*.bat$
Disallow: /*.py$
Disallow: /*.backup$

# ===========================================
# BAD BOTS (Block completely)
# ===========================================

# Block aggressive scrapers
User-agent: MegaIndex
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: PetalBot
Disallow: /

# Block email harvesters
User-agent: EmailCollector
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: HTTrack
Disallow: /

# ===========================================
# SITEMAP LOCATION
# ===========================================
Sitemap: https://algovesta.com/sitemap.xml

# ===========================================
# LLM / AI DISCOVERY (llms.txt — 13 dil)
# ===========================================
# AI motorlari (GPTBot, ClaudeBot, PerplexityBot, OAI-SearchBot...) siteyi anlamak icin
# llms.txt okuyabilir. AlgoVesta'yi tanimlayan ozet — MCP server for 16 exchanges + MT5,
# Telegram/TradingView sinyal otomasyonu — bu dosyalarin ilk satirlarindadir.
# Her dilin KENDI llms.txt'i vardir.
# LLM-Content: https://algovesta.com/llms.txt
# LLM-Full-Content: https://algovesta.com/llms-full.txt
# LLM-Content-tr: https://algovesta.com/tr/llms.txt
# LLM-Content-es: https://algovesta.com/es/llms.txt
# LLM-Content-pt: https://algovesta.com/pt/llms.txt
# LLM-Content-de: https://algovesta.com/de/llms.txt
# LLM-Content-fr: https://algovesta.com/fr/llms.txt
# LLM-Content-ru: https://algovesta.com/ru/llms.txt
# LLM-Content-ko: https://algovesta.com/ko/llms.txt
# LLM-Content-it: https://algovesta.com/it/llms.txt
# LLM-Content-ja: https://algovesta.com/ja/llms.txt
# LLM-Content-ar: https://algovesta.com/ar/llms.txt
# LLM-Content-zh: https://algovesta.com/zh/llms.txt
# LLM-Content-hi: https://algovesta.com/hi/llms.txt

# ===========================================
# HOST — KALDIRILDI (2026-08-21, #561b)
# ===========================================
# `Host:` direktifi RFC 9309'da YOKTUR; Yandex'e ozgu bir uzantiydi ve Yandex bunu
# 2021'de KULLANIMDAN KALDIRDI -> bugun hicbir arama motoru okumuyor.
# Bing robots.txt Tester bu satiri HATA olarak isaretliyordu ("1 errors, 0 warnings").
# Ustelik Yandex'in kendi kuralinda deger protokolsuz olmaliydi (Host: algovesta.com);
# 'https://' ile yazilmasi ikinci bir hataydi -> SIFIR fayda, BIR hata.
# Kanonik alan adi zaten her sayfada <link rel="canonical"> ile bildiriliyor.
# TEKRAR EKLEME.