# robots.txt for https://24toolkit.com
# Last updated: 2026-04-22
# Multilingual support: en, ar, fr, hi, de, es

# Global crawl policy (search indexing + normal crawlers)
User-agent: *
Allow: /
Allow: /en/
Allow: /ar/
Allow: /fr/
Allow: /hi/
Allow: /de/
Allow: /es/
Allow: /assets/
Allow: /images/
Disallow: /api/
Disallow: /admin/
Disallow: /_internal/
Disallow: /*?*utm_
Disallow: /*?*ref=
Disallow: /*?*source=
Disallow: /*?*search=
Disallow: /*?*category=
Disallow: /*?*filter=
Disallow: /*?*preview=
Content-Signal: ai-train=no, search=yes, ai-input=yes
Crawl-delay: 1

# Explicit AI search crawler policy (allowed)
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /admin/

# Training crawlers (restricted by policy)
User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: cohere-ai
Disallow: /

# Sitemap
Sitemap: https://24toolkit.com/sitemap.xml