# ================================================
# Builera - AI Prompt Generator for AI Builders
# https://builera.app
# ================================================

# ============ TRADITIONAL SEARCH ENGINES ============

User-agent: Googlebot
Allow: /
Allow: /features
Allow: /pricing
Allow: /about
Allow: /blog
Allow: /blog/*
Allow: /ai-builder
Allow: /ai-builder/lovable
Allow: /ai-builder/cursor
Allow: /ai-builder/v0
Allow: /ai-builder/bolt
Allow: /ai-builder/base44
Allow: /ai-builder/google-ai-studio
Allow: /lovable-vs-cursor
Allow: /lovable-vs-v0
Allow: /lovable-vs-bolt
Allow: /lovable-vs-base44
Allow: /lovable-vs-google-ai-studio
Disallow: /dashboard
Disallow: /create
Disallow: /project/
Disallow: /profile
Disallow: /admin
Disallow: /auth

User-agent: Bingbot
Allow: /
Allow: /features
Allow: /pricing
Allow: /about
Allow: /blog
Allow: /ai-builder
Allow: /ai-builder/*
Allow: /lovable-vs-*
Disallow: /dashboard
Disallow: /create
Disallow: /project/
Disallow: /profile
Disallow: /admin
Disallow: /auth

User-agent: Yandex
Allow: /
Allow: /ai-builder
Allow: /lovable-vs-*
Disallow: /dashboard
Disallow: /create
Disallow: /project/
Disallow: /profile
Disallow: /admin

User-agent: DuckDuckBot
Allow: /
Allow: /ai-builder
Allow: /lovable-vs-*
Disallow: /dashboard
Disallow: /create
Disallow: /project/
Disallow: /profile
Disallow: /admin

# ============ SOCIAL MEDIA CRAWLERS ============

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: WhatsApp
Allow: /

# ============ AI SEARCH & LLM CRAWLERS ============
# Full access to public content for AI Overviews (AIO)

# OpenAI
User-agent: GPTBot
Allow: /
Allow: /llms.txt
Allow: /ai-builder
Allow: /ai-builder/*
Allow: /lovable-vs-cursor
Allow: /lovable-vs-v0
Allow: /lovable-vs-bolt
Allow: /lovable-vs-base44
Allow: /lovable-vs-google-ai-studio
Disallow: /dashboard
Disallow: /create
Disallow: /project/
Disallow: /profile
Disallow: /admin

User-agent: ChatGPT-User
Allow: /

# Anthropic (Claude)
User-agent: Claude-Web
Allow: /

User-agent: Anthropic-AI
Allow: /
Allow: /llms.txt

User-agent: ClaudeBot
Allow: /
Allow: /llms.txt

# Perplexity
User-agent: PerplexityBot
Allow: /
Allow: /llms.txt

# Google AI (Gemini, Bard)
User-agent: Google-Extended
Allow: /
Allow: /llms.txt

# Cohere
User-agent: cohere-ai
Allow: /

# Common Crawl (used by many AI training)
User-agent: CCBot
Allow: /

# ByteDance AI
User-agent: Bytespider
Allow: /

# Apple AI
User-agent: Applebot-Extended
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# Microsoft AI (Copilot)
User-agent: bingbot
Allow: /

# You.com AI
User-agent: YouBot
Allow: /

# Neeva AI
User-agent: NeevaBot
Allow: /

# ============ DEFAULT RULE ============

User-agent: *
Allow: /
Allow: /features
Allow: /pricing
Allow: /about
Allow: /blog
Allow: /blog/*
Allow: /ai-builder
Allow: /ai-builder/lovable
Allow: /ai-builder/cursor
Allow: /ai-builder/v0
Allow: /ai-builder/bolt
Allow: /ai-builder/base44
Allow: /ai-builder/google-ai-studio
Allow: /lovable-vs-cursor
Allow: /lovable-vs-v0
Allow: /lovable-vs-bolt
Allow: /lovable-vs-base44
Allow: /lovable-vs-google-ai-studio
Disallow: /dashboard
Disallow: /create
Disallow: /project/
Disallow: /profile
Disallow: /admin
Disallow: /auth
Disallow: /api/

# ============ SITEMAPS ============

Sitemap: https://builera.app/sitemap.xml

# ============ ADDITIONAL FILES FOR AI ============
# llms.txt provides structured content for LLM crawlers
# See: https://builera.app/llms.txt