# =====================================================
# NicPouches.com — AI + SEO Optimised Robots.txt
# Updated: 2026-03-08
# =====================================================


# =====================================================
# OPENAI
# =====================================================

# ChatGPT Search — shows your pages as links inside ChatGPT search
User-agent: OAI-SearchBot
Allow: /

# User-driven browsing from ChatGPT and Custom GPTs
User-agent: ChatGPT-User
Allow: /

User-agent: ChatGPT-User/2.0
Allow: /

# Model training crawler (GPT-4o, GPT-5 etc.)
User-agent: GPTBot
Allow: /


# =====================================================
# ANTHROPIC (CLAUDE)
# =====================================================
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /


# =====================================================
# GOOGLE AI
# =====================================================

# Google AI products — Gemini, AI Overviews, Vertex AI
User-agent: Google-Extended
Allow: /

# Gemini Deep Research crawler
User-agent: Gemini-Deep-Research
Allow: /

# Google Shopping / product indexing
User-agent: Storebot-Google
Allow: /

# Google generic research crawler
User-agent: GoogleOther
Allow: /

# Google Cloud Vertex AI
User-agent: Google-CloudVertexBot
Allow: /


# =====================================================
# PERPLEXITY
# =====================================================
User-agent: PerplexityBot
Allow: /


# =====================================================
# APPLE
# =====================================================
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /


# =====================================================
# META
# =====================================================
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /


# =====================================================
# AMAZON / ALEXA
# =====================================================
User-agent: Amazonbot
Allow: /


# =====================================================
# COMMON CRAWL (used by many LLMs for training)
# =====================================================
User-agent: CCBot
Allow: /


# =====================================================
# COHERE
# =====================================================
User-agent: cohere-ai
Allow: /


# =====================================================
# YOU.COM
# =====================================================
User-agent: YouBot
Allow: /


# =====================================================
# DUCKDUCKGO AI
# =====================================================
User-agent: DuckAssistBot
Allow: /


# =====================================================
# DIFFBOT
# =====================================================
User-agent: Diffbot
Allow: /


# =====================================================
# Default rules for all other crawlers
# =====================================================
User-agent: *
Allow: /

# --- Block sensitive and system directories ---
Disallow: /admin
Disallow: /account
Disallow: /cart
Disallow: /carts
Disallow: /orders
Disallow: /checkouts/
Disallow: /checkout
Disallow: /57371230302/checkouts
Disallow: /57371230302/orders

# --- Block Shopify RSS/Atom feeds (crawl budget waste) ---
Disallow: /*.atom

# --- Block Shopify internal search tracking parameters ---
Disallow: /*?*_pos=*
Disallow: /*?*_fid=*
Disallow: /*?*_ss=*

# --- Block sitemap index URLs being crawled as pages ---
Disallow: /sitemap_pages_*.xml

# --- Prevent crawl bloat and duplicate URLs ---
Disallow: /collections/*sort_by*
Disallow: /*/collections/*sort_by*
Disallow: /collections/*+*
Disallow: /collections/*%2B*
Disallow: /*/collections/*+*
Disallow: /*/collections/*%2B*
Disallow: */collections/*filter*&*filter*
Disallow: /collections/*?filter*
Disallow: /*/collections/*?filter*
Disallow: /collections/vendors*
Disallow: /collections/tags*
Disallow: /collections/categories*
Disallow: /products/vendors*
Disallow: /products/category*
Disallow: /products/tags*
Disallow: /*?*filter=*
Disallow: /*?*sort_by=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /*/*?*ls=*&ls=*
Disallow: /*/*?*ls%3D*%3Fls%3D*
Disallow: /*/*?*ls%3d*%3fls%3d*

# --- Block non-content and utility endpoints ---
Disallow: /search
Disallow: /apple-app-site-association
Disallow: /.well-known/shopify/monorail
Disallow: /cdn/wpm/*.js
Disallow: /recommendations/products
Disallow: /*/recommendations/products

# --- Block policy pages (non-commercial) ---
Disallow: /policies/
Disallow: /*/policies/

# --- Sitemap ---
Sitemap: https://nicpouches.com/sitemap.xml


# =====================================================
# Google AdsBot (must be specifically declared)
# =====================================================
User-agent: adsbot-google
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts
Disallow: /orders
Disallow: /57371230302/checkouts
Disallow: /57371230302/orders
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /cdn/wpm/*.js


# =====================================================
# Aggressive or specialised crawlers
# =====================================================
User-agent: Nutch
Disallow: /

User-agent: AhrefsBot
Crawl-delay: 10
Disallow: /a/downloads/-/*
Disallow: /admin
Disallow: /cart
Disallow: /orders
Disallow: /checkouts/
Disallow: /checkout
Disallow: /57371230302/checkouts
Disallow: /57371230302/orders
Disallow: /carts
Disallow: /account
Disallow: /*.atom
Disallow: /*?*_pos=*
Disallow: /*?*_fid=*
Disallow: /*?*_ss=*
Disallow: /sitemap_pages_*.xml
Disallow: /collections/*sort_by*
Disallow: /*/collections/*sort_by*
Disallow: /collections/*+*
Disallow: /collections/*%2B*
Disallow: /*/collections/*+*
Disallow: /*/collections/*%2B*
Disallow: */collections/*filter*&*filter*
Disallow: /collections/*?filter*
Disallow: /*/collections/*?filter*
Disallow: /collections/vendors*
Disallow: /collections/tags*
Disallow: /collections/categories*
Disallow: /products/vendors*
Disallow: /products/category*
Disallow: /products/tags*
Disallow: /*?*filter=*
Disallow: /*?*sort_by=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /policies/
Disallow: /*/policies/
Disallow: /*/*?*ls=*&ls=*
Disallow: /*/*?*ls%3D*%3Fls%3D*
Disallow: /*/*?*ls%3d*%3fls%3d*
Disallow: /search
Disallow: /apple-app-site-association
Disallow: /.well-known/shopify/monorail
Disallow: /cdn/wpm/*.js
Disallow: /recommendations/products
Disallow: /*/recommendations/products

User-agent: AhrefsSiteAudit
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: Pinterest
Crawl-delay: 1