User-agent: *
Allow: /

# Enterprise solutions (high priority for B2B search)
Allow: /enterprise

# Allow all bots to index landing pages
Allow: /ai-hebrew-translator
Allow: /best-hebrew-translator
Allow: /best-iphone-hebrew-translator
Allow: /best-android-hebrew-translator
Allow: /most-accurate-hebrew-translator
Allow: /hebrew-translator-app
Allow: /free-hebrew-translator
Allow: /hebrew-english-translator
Allow: /best-hebrew-translators-guide
Allow: /learn-hebrew-app

# Feature-specific pages
Allow: /gendered-hebrew-translation
Allow: /hebrew-translator-with-camera
Allow: /hebrew-pdf-translation
Allow: /hebrew-slang-translator
Allow: /hebrew-text-to-speech

# Hebrew language page
Allow: /he

# Programmatic Hebrew phrases pages
Allow: /hebrew-phrases/

# Tool pages
Allow: /israel-time-converter
Allow: /hebrew-calculators
Allow: /hebrew-date-converter

# Press and resources
Allow: /press

# Block internal/test routes
Disallow: /admin
Disallow: /api/
# App hand-off links (https://itsbaba.com/app/...) only redirect to the stores
Disallow: /app/
Disallow: /appsflyer-test
Disallow: /appsflyer-debug
Disallow: /banner-test

# Blog — allow posts, block ghost/pagination internals
Allow: /blog/
Disallow: /ghost/
Disallow: /p/
Disallow: /blog/author/
Disallow: /blog/page/

# NOTE: do NOT block /_next/static/. Google needs to fetch the page's JS/CSS to
# render and evaluate it; blocking it hurts rendering and produced 120 "Indexed,
# though blocked by robots.txt" warnings on versioned chunk URLs. Let Google
# crawl them — they're non-HTML and drop out of the index on their own.

# AI search and assistant crawlers: explicitly allowed, in one group.
# A crawler that matches a named group ignores the * group entirely (RFC 9309
# section 2.2.1), so the private paths are repeated here. Before 2026-10-05 each
# bot had its own "Allow: /" group, which quietly let them into /api/ and /app/.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: Amazonbot
User-agent: CCBot
User-agent: Bytespider
User-agent: cohere-ai
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /app/
Disallow: /appsflyer-test
Disallow: /appsflyer-debug
Disallow: /banner-test
Disallow: /ghost/
Disallow: /p/
Disallow: /blog/author/
Disallow: /blog/page/

# Sitemap declaration
Sitemap: https://www.itsbaba.com/sitemap.xml
Sitemap: https://www.itsbaba.com/blog/sitemap.xml
Sitemap: https://www.itsbaba.com/podcast/sitemap.xml
Sitemap: https://www.itsbaba.com/sitemap-llms.xml
# Baba News lives on its own subdomain — its sitemaps are declared in
# https://news.itsbaba.com/robots.txt, which Google reads at that host root.