# ==============================================
# TakeHost.biz Robots.txt - Allow All Crawlers
# ==============================================

# Main directive - Allow ALL search engines and crawlers
User-agent: *
Allow: /
Crawl-delay: 0

# Explicitly allow all major pages for maximum indexing
Allow: /virtual-servers
Allow: /sweden-vps
Allow: /dedicated-servers
Allow: /10gbps-dedicated-servers
Allow: /storage-servers
Allow: /web-hosting
Allow: /mail-hosting
Allow: /game-servers
Allow: /game-servers/minecraft
Allow: /game-servers/counter-strike-2
Allow: /game-servers/counter-strike-16
Allow: /game-servers/fivem
Allow: /network
Allow: /about
Allow: /tshield
Allow: /status
Allow: /privacy
Allow: /terms
Allow: /cookies
Allow: /promotions/autumn2025
Allow: /documentation
Allow: /tutorials
Allow: /fiber-internet

# Block ONLY admin and API areas
Disallow: /adminedittake32
Disallow: /admin
Disallow: /manage
Disallow: /api/

# Allow Next.js static files for proper rendering
Allow: /_next/static/
Allow: /_next/image

# Sitemap locations (both www and non-www)
Sitemap: https://www.takehost.biz/sitemap.xml
Sitemap: https://takehost.biz/sitemap.xml

# ==============================================
# Specific Bot Configurations
# ==============================================

# Google Bot - Maximum access, no restrictions
User-agent: Googlebot
Allow: /
Crawl-delay: 0
Disallow: /adminedittake32
Disallow: /admin
Disallow: /manage
Disallow: /api/

# Google Image Bot
User-agent: Googlebot-Image
Allow: /
Crawl-delay: 0

# Google Mobile Bot
User-agent: Googlebot-Mobile
Allow: /
Crawl-delay: 0

# Bing Bot - Maximum access
User-agent: Bingbot
Allow: /
Crawl-delay: 0
Disallow: /adminedittake32
Disallow: /admin
Disallow: /manage
Disallow: /api/

# Bing Preview Bot
User-agent: BingPreview
Allow: /
Crawl-delay: 0

# Yandex Bot - Russian search engine
User-agent: Yandex
Allow: /
Crawl-delay: 0
Disallow: /adminedittake32
Disallow: /admin
Disallow: /manage
Disallow: /api/

# Baidu Bot - Chinese search engine
User-agent: Baiduspider
Allow: /
Crawl-delay: 0
Disallow: /adminedittake32
Disallow: /admin
Disallow: /manage
Disallow: /api/

# DuckDuckGo Bot
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 0

# Yahoo Bot
User-agent: Slurp
Allow: /
Crawl-delay: 0

# Ask Jeeves Bot
User-agent: Teoma
Allow: /
Crawl-delay: 0

# AOL Bot
User-agent: ia_archiver
Allow: /
Crawl-delay: 0

# Alexa Bot
User-agent: ia_archiver-web.archive.org
Allow: /
Crawl-delay: 0

# Facebook Bot (for link previews)
User-agent: facebookexternalhit
Allow: /
Crawl-delay: 0

# Twitter Bot (for card previews)
User-agent: Twitterbot
Allow: /
Crawl-delay: 0

# LinkedIn Bot
User-agent: LinkedInBot
Allow: /
Crawl-delay: 0

# Pinterest Bot
User-agent: Pinterest
Allow: /
Crawl-delay: 0

# Telegram Bot
User-agent: TelegramBot
Allow: /
Crawl-delay: 0

# Discord Bot
User-agent: Discordbot
Allow: /
Crawl-delay: 0

# Slack Bot
User-agent: Slackbot
Allow: /
Crawl-delay: 0

# WhatsApp Bot
User-agent: WhatsApp
Allow: /
Crawl-delay: 0

# Apple Bot (for Siri and Spotlight)
User-agent: Applebot
Allow: /
Crawl-delay: 0

# Samsung Bot
User-agent: SamsungBot
Allow: /
Crawl-delay: 0

# Sogou Bot (Chinese search engine)
User-agent: Sogou
Allow: /
Crawl-delay: 0

# Exabot (European search engine)
User-agent: Exabot
Allow: /
Crawl-delay: 0

# ==============================================
# End of robots.txt
# ==============================================