# ========================================
# ROBOTS.TXT - OPTIMIZED FOR SEO & SECURITY
# ========================================

# ========================================
# ALLOWED BOTS - ESSENTIAL FOR SEO
# ========================================

# Google Search Bot (MOST IMPORTANT)
User-agent: Googlebot
Allow: /
Crawl-delay: 1

# Google Image Bot
User-agent: Googlebot-Image
Allow: /
Crawl-delay: 1

# Google Mobile Bot
User-agent: Googlebot-Mobile
Allow: /
Crawl-delay: 1

# Bing Search Bot
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Bing Preview Bot (for social sharing)
User-agent: BingPreview
Allow: /

# Yandex Bot
User-agent: Yandex
Allow: /
Crawl-delay: 2

# DuckDuckGo Bot
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

# Baidu Bot
User-agent: Baiduspider
Allow: /
Crawl-delay: 2

# Sogou Bot
User-agent: Sogou
Allow: /
Crawl-delay: 2

# ========================================
# SOCIAL MEDIA BOTS - ALLOW FOR SHARING
# ========================================

# Facebook External Hit (for social previews)
User-agent: facebookexternalhit
Allow: /

# Twitter Bot
User-agent: Twitterbot
Allow: /

# LinkedIn Bot
User-agent: LinkedInBot
Allow: /

# Pinterest Bot
User-agent: Pinterest
Allow: /

# ========================================
# ESSENTIAL TOOLS - ALLOW FOR DEVELOPMENT
# ========================================

# Google PageSpeed Insights
User-agent: Google PageSpeed Insights
Allow: /

# Google Structured Data Testing Tool
User-agent: Google-Structured-Data-Testing-Tool
Allow: /

# Google Web Preview
User-agent: Google Web Preview
Allow: /

# ========================================
# BLOCK AI TRAINING BOTS
# ========================================

# Anthropic AI
User-agent: anthropic-ai
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: Claude-Web
Disallow: /

# OpenAI
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: GPT-4
Disallow: /
User-agent: GPT-4o
Disallow: /
User-agent: GPT-3
Disallow: /
User-agent: GPT-3.5
Disallow: /

# Google AI
User-agent: Google-Extended
Disallow: /
User-agent: GoogleOther
Disallow: /
User-agent: Gemini
Disallow: /
User-agent: GeminiBot
Disallow: /

# Apple AI
User-agent: Applebot-Extended
Disallow: /

# Perplexity AI
User-agent: PerplexityBot
Disallow: /
User-agent: Perplexity
Disallow: /

# You.com
User-agent: YouBot
Disallow: /
User-agent: You.com
Disallow: /

# Meta/Facebook AI
User-agent: Meta-ExternalAgent
Disallow: /
User-agent: FacebookBot
Disallow: /

# ByteDance/TikTok AI
User-agent: Bytespider
Disallow: /

# Common Crawl
User-agent: CCBot
Disallow: /

# Cohere AI
User-agent: cohere-ai
Disallow: /

# DataForSeoBot
User-agent: DataForSeoBot
Disallow: /

# Diffbot
User-agent: Diffbot
Disallow: /

# Amazon AI
User-agent: Amazonbot
Disallow: /

# Yandex AI
User-agent: YandexGPT
Disallow: /

# ========================================
# BLOCK AGGRESSIVE SEO TOOLS
# ========================================

# Ahrefs
User-agent: AhrefsBot
Disallow: /

# SEMrush
User-agent: SemrushBot
Disallow: /

# Screaming Frog
User-agent: Screaming Frog SEO Spider
Disallow: /

# Xenu Link Sleuth
User-agent: Xenu Link Sleuth
Disallow: /

# ========================================
# BLOCK SCRAPERS & CRAWLERS
# ========================================

# Scrapy
User-agent: Scrapy
Disallow: /

# PhantomJS
User-agent: PhantomJS
Disallow: /

# Curl
User-agent: curl
Disallow: /

# Wget
User-agent: wget
Disallow: /

# Python Requests
User-agent: python-requests
Disallow: /

User-agent: python-httplib2
Disallow: /

User-agent: python-urllib
Disallow: /

# ========================================
# BLOCK MONITORING BOTS (OPTIONAL)
# ========================================

# Pingdom
User-agent: Pingdom
Disallow: /

# UptimeRobot
User-agent: UptimeRobot
Disallow: /

# StatusCake
User-agent: StatusCake
Disallow: /

# ========================================
# BLOCK OTHER UNWANTED BOTS
# ========================================

# Archive.org (if you don't want archiving)
User-agent: ia_archiver
Disallow: /

# Common Crawl
User-agent: CCBot
Disallow: /

# ========================================
# BLOCK CLOUD & HOSTING PROVIDER BOTS
# ========================================

# Google Cloud
User-agent: Google-Cloud
Disallow: /
User-agent: GoogleCloud
Disallow: /
User-agent: Google-private-cloud
Disallow: /
User-agent: GoogleProxy
Disallow: /
User-agent: Google Proxy
Disallow: /

# Microsoft Corporation
User-agent: MicrosoftURLControl
Disallow: /
User-agent: Microsoft Office Protocol Discovery
Disallow: /
User-agent: Microsoft-WebDAV-MiniRedir
Disallow: /

# OVH SAS
User-agent: OVH
Disallow: /
User-agent: OVH SAS
Disallow: /

# DigitalOcean
User-agent: DigitalOcean
Disallow: /

# Huawei Clouds
User-agent: HuaweiClouds
Disallow: /
User-agent: HuaweiCloud
Disallow: /
User-agent: Huawei
Disallow: /

# Facebook (chặn toàn bộ bot Facebook ngoài bot social sharing)
User-agent: Facebook
Disallow: /
User-agent: Facebot
Disallow: /

# Amazon.com
User-agent: Amazonbot
Disallow: /
User-agent: Amazon-Route53-Health-Check-Service
Disallow: /
User-agent: AWS
Disallow: /
User-agent: Amazon
Disallow: /

# Omonia d.o.o.
User-agent: Omonia
Disallow: /
User-agent: Omonia d.o.o.
Disallow: /

# ColoCrossing
User-agent: ColoCrossing
Disallow: /

# ========================================
# BLOCK CLICK FRAUD, FAKE TRAFFIC, ANALYTIC SPAM BOTS
# ========================================

User-agent: Clickagy
Disallow: /
User-agent: ClickCease
Disallow: /
User-agent: ClickGuard
Disallow: /
User-agent: Clixtell
Disallow: /
User-agent: AdbeatBot
Disallow: /
User-agent: AdMuncher
Disallow: /
User-agent: AdRecon
Disallow: /
User-agent: AdScanner
Disallow: /
User-agent: AdSpider
Disallow: /
User-agent: Semalt
Disallow: /
User-agent: Darodar
Disallow: /
User-agent: Buttons-for-website
Disallow: /
User-agent: SimplePie
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: MegaIndex.ru
Disallow: /
User-agent: RankActiveLinkBot
Disallow: /
User-agent: RankFlex
Disallow: /
User-agent: RankSonicSiteAuditor
Disallow: /
User-agent: SeznamBot
Disallow: /
User-agent: spbot
Disallow: /
User-agent: SputnikBot
Disallow: /
User-agent: YandexImages
Disallow: /
User-agent: YandexMetrika
Disallow: /
User-agent: Sogou web spider
Disallow: /
User-agent: Sogou inst spider
Disallow: /
User-agent: Sogou spider2
Disallow: /
User-agent: Sogou blog
Disallow: /
User-agent: Sogou News Spider
Disallow: /
User-agent: Sogou Orion spider
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: Exabot
Disallow: /
User-agent: Barkrowler
Disallow: /
User-agent: ZoominfoBot
Disallow: /
User-agent: MauiBot
Disallow: /
User-agent: Seekport Crawler
Disallow: /
User-agent: SeekportBot
Disallow: /
User-agent: NetcraftSurveyAgent
Disallow: /
User-agent: NetEstate NE Crawler
Disallow: /
User-agent: SISTRIX Crawler
Disallow: /
User-agent: SISTRIX
Disallow: /
User-agent: magpie-crawler
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: KagiBot
Disallow: /
User-agent: KagiSearch
Disallow: /
User-agent: GigaBot
Disallow: /
User-agent: CensysInspect
Disallow: /