# robots.txt for https://www.nipaers.com
# Last reviewed: 2026-07-09
# Review this file quarterly — AI crawler user-agents change 2-4x per year.

# =====================================================
# STANDARD SEARCH ENGINES — full access
# =====================================================
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

# =====================================================
# AI SEARCH / RETRIEVAL / USER-TRIGGERED AGENTS
# Allow these — they are what get you CITED in ChatGPT,
# Claude, Perplexity, and Gemini answers. Blocking them
# removes you from AI-generated answers entirely.
# =====================================================

# OpenAI — real-time search indexing + user-triggered fetch
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic — citation/search + user-triggered fetch
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# DuckDuckGo AI Assist
User-agent: DuckAssistBot
Allow: /

# Mistral
User-agent: MistralAI-User
Allow: /


# =====================================================
# DEFAULT RULE FOR EVERYTHING ELSE
# =====================================================
User-agent: *
Allow: /

# Block low-value / duplicate-content paths (adjust to your actual URL structure)
Disallow: /cart
Disallow: /checkout
Disallow: /account
Disallow: /wishlist
Disallow: /search
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?utm_
Disallow: /admin
Disallow: /*.pdf$

# =====================================================
# SITEMAPS
# =====================================================
Sitemap: https://www.nipaers.com/sitemap.xml
Sitemap: https://www.nipaers.com/sitemap-books.xml