# robots.txt for temp-mail.lol - Optimized for SEO
# Last updated: 2025-08-16

# Default rules for all crawlers
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /*.json$
Disallow: /dist/
Disallow: /node_modules/
Disallow: /.git/

# High-priority crawl bots with optimized settings
User-agent: Googlebot
Crawl-delay: 1
Allow: /
Allow: /*.js$
Allow: /*.css$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.webp$
Allow: /*.svg$
Allow: /*.woff$
Allow: /*.woff2$
Allow: /*.ico$

User-agent: Bingbot
Crawl-delay: 1
Allow: /

# Google's specialized crawlers
User-agent: Googlebot-Image
Allow: /
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.webp$
Allow: /*.svg$

User-agent: Googlebot-Mobile
Allow: /

User-agent: Googlebot-News
Allow: /blog/
Allow: /

User-agent: Googlebot-Video
Disallow: /

# Social media crawlers
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot  
Allow: /

User-agent: LinkedInBot
Allow: /

# LLM and AI Crawlers - Allow for training and context
User-agent: GPTBot
Allow: /
Allow: /llm.txt

User-agent: ChatGPT-User
Allow: /
Allow: /llm.txt

User-agent: Claude-Web
Allow: /
Allow: /llm.txt

User-agent: anthropic-ai
Allow: /
Allow: /llm.txt

User-agent: Google-Extended
Allow: /
Allow: /llm.txt

User-agent: CCBot
Allow: /
Allow: /llm.txt

User-agent: Omgilibot
Allow: /
Allow: /llm.txt

User-agent: FacebookBot
Allow: /
Allow: /llm.txt

User-agent: Applebot-Extended
Allow: /
Allow: /llm.txt

User-agent: PerplexityBot
Allow: /
Allow: /llm.txt

User-agent: Diffbot
Allow: /
Allow: /llm.txt

User-agent: cohere-ai
Allow: /
Allow: /llm.txt

# Block aggressive crawlers that can impact performance
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

# Sitemap locations
Sitemap: https://temp-mail.lol/sitemap.xml
Sitemap: https://temp-mail.lol/sitemap-index.xml

# LLM context file for AI agents
# Learn more: https://llmstxt.org/
LLM: https://temp-mail.lol/llm.txt

# Default crawl delay for other bots
Crawl-delay: 3

# Host directive to specify preferred domain
Host: https://temp-mail.lol