# Robots.txt for NanoTech Materials
# nanotechmaterials.com

# ─────────────────────────────────────────────────────────────────────
# Default: allow all standard crawlers (Googlebot, Bingbot, DuckDuckBot,
# YandexBot, etc.) with admin / api / internals blocked.
# ─────────────────────────────────────────────────────────────────────
User-agent: *
Disallow: /admin
Disallow: /api/
Disallow: /cdn-cgi/
Disallow: /_next/
Allow: /_next/static/
Allow: /_next/image

# ─────────────────────────────────────────────────────────────────────
# AI search citation crawlers — explicitly allowed.
# These bots fetch pages in real time to ground AI search answers and
# cite NanoTech Materials as a source. Same surface area as Googlebot.
# Policy decision: 2026-05-13 (SEO Audit Phase 0).
# ─────────────────────────────────────────────────────────────────────
User-agent: PerplexityBot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: OAI-SearchBot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: ChatGPT-User
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: ClaudeBot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: Claude-Web
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: Applebot-Extended
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: Google-Extended
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /cdn-cgi/

# ─────────────────────────────────────────────────────────────────────
# AI training-only crawlers — disallowed.
# These crawlers harvest content for model training rather than
# real-time citation. Disallowing them preserves leverage over
# how NanoTech's technical content is used in commercial training sets.
# Policy decision: 2026-05-13 (SEO Audit Phase 0).
# ─────────────────────────────────────────────────────────────────────
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

# ─────────────────────────────────────────────────────────────────────
# Sitemap
# ─────────────────────────────────────────────────────────────────────
Sitemap: https://nanotechmaterials.com/sitemap.xml