# Robots.txt for www.notificationapi.com
# Updated for better SEO and crawler management

# Allow all crawlers access to public content
User-agent: *
Allow: /

# Prevent indexing of search results or internal pages
Disallow: /search*
Disallow: /_*
Disallow: /admin*
Disallow: /*.json$
Disallow: /*.xml$

# Block malformed docs URLs and component files
Disallow: /docs/*/docs/
Disallow: /docs/components/*.astro
Disallow: /**/NotificationAPI$
Disallow: /docs/*/NotificationAPI$

# Respect crawl delay to avoid overwhelming the server
Crawl-delay: 1

# Specific rules for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: facebookexternalhit
Allow: /

# Block aggressive crawlers that don't respect rate limits
User-agent: SemrushBot
Disallow: /

# Allow AhrefsBot for SEO monitoring but with crawl delay
User-agent: AhrefsBot
Allow: /
Crawl-delay: 2

User-agent: MJ12bot
Disallow: /

# Sitemap location
Sitemap: https://www.notificationapi.com/sitemap-index.xml

# Note: The app subdomain (app.notificationapi.com) has its own robots.txt
# that blocks all crawlers since it contains private application content