User-agent: *
Allow: /

# AI Crawlers - Explicitly allow
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Bytespider
Allow: /

# Additional AI Crawlers - Added 2025-09-29
User-agent: cohere-ai
Allow: /

User-agent: Omgilibot
Allow: /

User-agent: Omgili
Allow: /

User-agent: YouBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: Diffbot
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: ImagesiftBot
Allow: /

User-agent: img2dataset
Allow: /

# Block access to WordPress admin and includes (legacy files)
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-includes/

# Block WordPress RSS feeds (legacy - no longer available)
Disallow: /*?feed=
Disallow: /*&feed=

# Block access to sensitive files
Disallow: /wp-config.php
Disallow: /readme.html
Disallow: /license.txt

# Block backup files and directories

# Demo pages - not for public indexing
Disallow: /EnergyOptimizer/
Disallow: /BACKUP-*/
Disallow: *.tar.gz
Disallow: *.backup
Disallow: *.bak
Disallow: *.sh

# Block markdown documentation files (not for public crawling)
Disallow: /*.md$
Disallow: /nextjs-rebuild/


# Allow important files explicitly
Allow: /case-study.pdf

# Allow important HTML pages explicitly
Allow: /index.html
Allow: /contact.html
Allow: /careers.html
Allow: /demo.html
Allow: /get-started.html
Allow: /404.html

# Sitemap location
Sitemap: https://factorydb.io/sitemap.xml

# Crawl delay for aggressive bots (prevents server overload)
User-agent: *
Crawl-delay: 1