# robots.txt for Hero Forge - AI Fantasy Character Generator
# This file tells search engines how to crawl and index this website
# Website: https://hero-forge-98367.web.app

# Default rule for all crawlers
User-agent: *
Allow: /

# Sitemap locations
Sitemap: https://www.hero-forge.com/sitemap.xml

# Major search engines - explicit permissions
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

User-agent: Baiduspider
Allow: /
Crawl-delay: 2

User-agent: YandexBot
Allow: /
Crawl-delay: 2

# Social media crawlers
User-agent: facebookexternalhit
Allow: /
Crawl-delay: 1

User-agent: Twitterbot
Allow: /
Crawl-delay: 1

User-agent: LinkedInBot
Allow: /
Crawl-delay: 1

# Additional social media and messaging platforms
User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: SkypeUriPreview
Allow: /

User-agent: Applebot
Allow: /
Crawl-delay: 1

# Ad network crawlers (for ads.txt verification)
User-agent: Google-AdSense
Allow: /ads.txt
Allow: /

User-agent: Google-Ads-Creatives-Assistant
Allow: /

# SEO and analytics crawlers
User-agent: AhrefsBot
Allow: /
Crawl-delay: 10

User-agent: SemrushBot
Allow: /
Crawl-delay: 10

User-agent: MJ12bot
Allow: /
Crawl-delay: 10

# Disallow paths that shouldn't be indexed
Disallow: /api/
Disallow: /.well-known/
Disallow: /test-*
Disallow: /*?source=twa
Disallow: /proxy-server/
Disallow: /node_modules/
Disallow: /build/
Disallow: /dist/
Disallow: /.git/

# Explicitly allow important file types for SEO and functionality
Allow: *.css$
Allow: *.js$
Allow: *.png$
Allow: *.jpg$
Allow: *.jpeg$
Allow: *.gif$
Allow: *.webp$
Allow: *.svg$
Allow: *.ico$
Allow: *.woff$
Allow: *.woff2$
Allow: *.ttf$
Allow: *.eot$
Allow: *.json$
Allow: *.xml$
Allow: *.txt$
Allow: *.pdf$

# Allow important files
Allow: /manifest.json
Allow: /ads.txt
Allow: /sitemap.xml
Allow: /robots.txt
Allow: /favicon.ico
Allow: /googlee85c662c8a193daa.html
Allow: /.well-known/assetlinks.json

# Block aggressive crawlers that might impact performance
User-agent: AhrefsBot
Crawl-delay: 30

User-agent: SemrushBot
Crawl-delay: 30

User-agent: MJ12bot
Crawl-delay: 30

User-agent: DotBot
Crawl-delay: 30

# Block known bad bots
User-agent: SiteBot
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft URL Control
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: WebBandit
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: CopyRightCheck
Disallow: /

User-agent: crescent
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: ProWebWalker
Disallow: /

User-agent: CheeseBot
Disallow: /

User-agent: LNSpiderguy
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: ia_archiver/1.6
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: MIIxpc
Disallow: /

User-agent: Telesoft
Disallow: /

User-agent: Website Quester
Disallow: /

User-agent: moget/2.1
Disallow: /

User-agent: WebZip/4.0
Disallow: /

User-agent: Stock Photo Downloader v1.0
Disallow: /

# Additional information
# Website: https://hero-forge-98367.web.app
# Contact: jonslavey@hotmail.com
# Description: AI-powered fantasy character generator for RPGs and D&D
# Purpose: Create stunning fantasy characters for tabletop RPGs, D&D campaigns, and creative projects
# Features: Character generation, artifact creation, banner design, animation
# Last updated: 2025-10-06