# Digital Magma | https://digitalmagma.com/
# Crawl policy reviewed: 2026-10-11
# Public services, locations, portfolio, case studies, blog, images, CSS and JS are crawlable.
# Named groups below share identical exclusions; other bots use the wildcard group.
# Training and dataset crawling remain allowed under the owner's broad-access preference.
# Permission to crawl does not guarantee indexing, citations, rankings or referral traffic.
# robots.txt is voluntary crawl guidance, not authentication or data protection.
# No Crawl-delay, blanket query-string block or unsupported Noindex directive.
# AI content guide (informational link, not a robots directive): https://digitalmagma.com/llms.txt

# Default: all other compliant crawlers, including social preview and new AI bots
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# ============================================================
# Search engines and search rendering
# ============================================================

# Google Search, including AI Overviews and AI Mode
User-agent: Googlebot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Google Images
User-agent: Googlebot-Image
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Google video discovery
User-agent: Googlebot-Video
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Google Search testing tools
User-agent: Google-InspectionTool
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Bing indexing; relevant to Bing and Copilot discovery
User-agent: bingbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Microsoft page previews
User-agent: MicrosoftPreview
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# DuckDuckGo search indexing
User-agent: DuckDuckBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Apple search products
User-agent: Applebot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# ============================================================
# AI search and answer discovery
# ============================================================

# ChatGPT search indexing; distinct from GPTBot training
User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Claude search discovery
User-agent: Claude-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Perplexity search discovery
User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# DuckDuckGo AI-assisted answers
User-agent: DuckAssistBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Amazon search and Alexa discovery; not model training
User-agent: Amzn-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# ============================================================
# User-requested retrieval (some fetchers may ignore robots.txt)
# ============================================================

# ChatGPT user-requested page fetches; not search indexing
User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Claude user-requested page fetches
User-agent: Claude-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Perplexity user-requested page fetches
User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Amazon and Alexa user-requested page fetches
User-agent: Amzn-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# ============================================================
# Advertising landing-page checks
# ============================================================

# Google Ads desktop landing-page checks
User-agent: AdsBot-Google
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Google Ads mobile landing-page checks
User-agent: AdsBot-Google-Mobile
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Checks pages submitted as ChatGPT ads; not organic indexing
User-agent: OAI-AdsBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# ============================================================
# Model development and open datasets (allowed; no ranking benefit promised)
# ============================================================

# OpenAI model-development crawler
User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Anthropic model-development crawler
User-agent: ClaudeBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Gemini training and grounding control token; not a Search ranking signal
User-agent: Google-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Apple model-training control token
User-agent: Applebot-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Amazon crawler with potential model-training use
User-agent: Amazonbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Common Crawl public web dataset
User-agent: CCBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Meta AI crawler token documented in Cloudflare crawler controls
User-agent: meta-externalagent
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# ============================================================
# Other documented crawling and SEO tools
# ============================================================

# Google non-Search research crawling
User-agent: GoogleOther
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Ahrefs web index and Yep; not a Google ranking signal
User-agent: AhrefsBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Owner-requested Ahrefs technical site audits
User-agent: AhrefsSiteAudit
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Semrush backlink and website analysis
User-agent: SemrushBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /xmlrpc.php
Disallow: /wp-content/dm-backups/
Disallow: /wp-content/upgrade/

# Canonical sitemap index (includes page, post and service sitemaps)
Sitemap: https://digitalmagma.com/sitemap_index.xml