# robots.txt — w3developing.com
# Crawl everything public; keep admin/internal endpoints out of the index.

User-agent: *
# Cloudflare/IETF content signals — the machine-readable form of the same
# policy: index it, quote it when answering someone, do not train on it.
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

# Internal tooling / non-indexable endpoints
Disallow: /admin.php
Disallow: /admin
Disallow: /stats.php
Disallow: /stats
Disallow: /track.php
Disallow: /proxy.php
Disallow: /update_order.php
Disallow: /og-image.php
Disallow: /optimize_tool.php
Disallow: /list_models.php
Disallow: /send_mail_simple.php
Disallow: /php/
Disallow: /cgi-bin/
Disallow: /dev/
Disallow: /data/

# ── AI crawlers ──────────────────────────────────────────────────────────────
# The song pages reserve this work against AI/ML training in words. This is the
# same reservation in the form crawlers actually read.
#
# The split below is deliberate, and it is not "block all AI". Two different
# things wear the same label:
#
#   · TRAINING crawlers take the work to build a model. Nothing comes back —
#     no link, no credit, no listener. Those are refused.
#   · ASSISTANT / SEARCH agents fetch a page to answer someone's question and
#     cite it, the way a search engine does. Those are welcomed: being findable
#     when someone asks about 01DW3ST is the entire point of publishing.
#
# Several vendors run one of each, which is why some names are allowed and
# similar-looking ones are not. Content-Signal (below) states the same policy
# declaratively for crawlers that honour it.

# Training — refused
User-agent: GPTBot
User-agent: Google-Extended
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: Amazonbot
User-agent: Bytespider
User-agent: CCBot
User-agent: cohere-ai
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Omgilibot
User-agent: Omgili
User-agent: PerplexityBot
User-agent: Timpibot
User-agent: YouBot
User-agent: AI2Bot
User-agent: Kangaroo Bot
User-agent: Scrapy
Disallow: /

# Answering a person's question, with a citation back — allowed, same rules as
# any search engine. These fetch on demand; they do not harvest for training.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Perplexity-User
User-agent: DuckAssistBot
Allow: /
Disallow: /admin.php
Disallow: /admin
Disallow: /stats.php
Disallow: /stats
Disallow: /php/
Disallow: /dev/
Disallow: /data/

Sitemap: https://w3developing.com/sitemap.xml